ReactOS  0.4.11-dev-433-g473ca91
intrin_x86.h
Go to the documentation of this file.
1 /*
2  Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3  Microsoft Visual C++ functions. Originally developed for the ReactOS
4  (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5  projects.
6 
7  Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8 
9  Permission is hereby granted, free of charge, to any person obtaining a
10  copy of this software and associated documentation files (the "Software"),
11  to deal in the Software without restriction, including without limitation
12  the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  and/or sell copies of the Software, and to permit persons to whom the
14  Software is furnished to do so, subject to the following conditions:
15 
16  The above copyright notice and this permission notice shall be included in
17  all copies or substantial portions of the Software.
18 
19  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  DEALINGS IN THE SOFTWARE.
26 */
27 
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30 
31 /*
32  FIXME: review all "memory" clobbers, add/remove to match Visual C++
33  behavior: some "obvious" memory barriers are not present in the Visual C++
34  implementation - e.g. __stosX; on the other hand, some memory barriers that
35  *are* present could have been missed
36 */
37 
38 /*
39  NOTE: this is a *compatibility* header. Some functions may look wrong at
40  first, but they're only "as wrong" as they would be on Visual C++. Our
41  priority is compatibility
42 
43  NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44  constraints and the uses of __volatile__ out of my... hat. Do not touch
45  them. I hate cargo cult programming
46 
47  NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48  barriers aren't there in Visual C++ (e.g. __stosX)
49 
50  NOTE: review all intrinsics with a return value, add/remove __volatile__
51  where necessary. If an intrinsic whose value is ignored generates a no-op
52  under Visual C++, __volatile__ must be omitted; if it always generates code
53  (for example, if it has side effects), __volatile__ must be specified. GCC
54  will only optimize out non-volatile asm blocks with outputs, so input-only
55  blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56  and follow Visual C++ behavior
57 
58  NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59  atomic operations. Test the version like this:
60 
61  #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62  ...
63 
64  Pay attention to the type of barrier. Make it match with what Visual C++
65  would use in the same case
66 */
67 
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71 
72 /*** memcopy must be memmove ***/
73 void* __cdecl memmove(void* dest, const void* source, size_t num);
74 __INTRIN_INLINE void* __cdecl memcpy(void* dest, const void* source, size_t num)
75 {
76  return memmove(dest, source, num);
77 }
78 
79 
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
84 
85 /* Maybe the same for x86? */
86 #ifdef __x86_64__
87 #define _alloca(s) __builtin_alloca(s)
88 #endif
89 
90 /*** Memory barriers ***/
91 
92 #if !HAS_BUILTIN(_ReadWriteBarrier)
94 {
95  __asm__ __volatile__("" : : : "memory");
96 }
97 #endif
98 
99 /* GCC only supports full barriers */
100 #define _ReadBarrier _ReadWriteBarrier
101 #define _WriteBarrier _ReadWriteBarrier
102 
103 #if !HAS_BUILTIN(_mm_mfence)
105 {
106  __asm__ __volatile__("mfence" : : : "memory");
107 }
108 #endif
109 
110 #if !HAS_BUILTIN(_mm_lfence)
112 {
113  _ReadBarrier();
114  __asm__ __volatile__("lfence");
115  _ReadBarrier();
116 }
117 #endif
118 
119 #if !HAS_BUILTIN(_mm_sfence)
121 {
122  _WriteBarrier();
123  __asm__ __volatile__("sfence");
124  _WriteBarrier();
125 }
126 #endif
127 
128 #ifdef __x86_64__
129 __INTRIN_INLINE void __faststorefence(void)
130 {
131  long local;
132  __asm__ __volatile__("lock; orl $0, %0;" : : "m"(local));
133 }
134 #endif
135 
136 
137 /*** Atomic operations ***/
138 
139 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
140 
141 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
142 {
143  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
144 }
145 
146 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
147 {
148  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
149 }
150 
151 #if !HAS_BUILTIN(_InterlockedCompareExchange)
152 __INTRIN_INLINE long __cdecl _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
153 {
154  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
155 }
156 #endif
157 
158 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
159 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
160 {
161  return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
162 }
163 #endif
164 
165 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
166 {
167  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
168  __sync_synchronize();
169  return __sync_lock_test_and_set(Target, Value);
170 }
171 
172 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
173 {
174  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
175  __sync_synchronize();
176  return __sync_lock_test_and_set(Target, Value);
177 }
178 
179 #if !HAS_BUILTIN(_InterlockedExchange)
180 __INTRIN_INLINE long __cdecl _InterlockedExchange(volatile long * Target, long Value)
181 {
182  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
183  __sync_synchronize();
184  return __sync_lock_test_and_set(Target, Value);
185 }
186 #endif
187 
188 #if !HAS_BUILTIN(_InterlockedExchangePointer)
189 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
190 {
191  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
192  __sync_synchronize();
193  return (void *)__sync_lock_test_and_set(Target, Value);
194 }
195 #endif
196 
197 #if defined(__x86_64__)
198 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * Target, long long Value)
199 {
200  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
201  __sync_synchronize();
202  return __sync_lock_test_and_set(Target, Value);
203 }
204 #endif
205 
206 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
207 {
208  return __sync_fetch_and_add(Addend, Value);
209 }
210 
211 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
212 {
213  return __sync_fetch_and_add(Addend, Value);
214 }
215 
216 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
217 __INTRIN_INLINE long __cdecl _InterlockedExchangeAdd(volatile long * Addend, long Value)
218 {
219  return __sync_fetch_and_add(Addend, Value);
220 }
221 #endif
222 
223 #if defined(__x86_64__)
224 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * Addend, long long Value)
225 {
226  return __sync_fetch_and_add(Addend, Value);
227 }
228 #endif
229 
230 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
231 {
232  return __sync_fetch_and_and(value, mask);
233 }
234 
235 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
236 {
237  return __sync_fetch_and_and(value, mask);
238 }
239 
240 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
241 {
242  return __sync_fetch_and_and(value, mask);
243 }
244 
245 #if defined(__x86_64__)
246 __INTRIN_INLINE long long _InterlockedAnd64(volatile long long * value, long long mask)
247 {
248  return __sync_fetch_and_and(value, mask);
249 }
250 #endif
251 
252 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
253 {
254  return __sync_fetch_and_or(value, mask);
255 }
256 
257 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
258 {
259  return __sync_fetch_and_or(value, mask);
260 }
261 
262 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
263 {
264  return __sync_fetch_and_or(value, mask);
265 }
266 
267 #if defined(__x86_64__)
268 __INTRIN_INLINE long long _InterlockedOr64(volatile long long * value, long long mask)
269 {
270  return __sync_fetch_and_or(value, mask);
271 }
272 #endif
273 
274 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
275 {
276  return __sync_fetch_and_xor(value, mask);
277 }
278 
279 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
280 {
281  return __sync_fetch_and_xor(value, mask);
282 }
283 
284 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
285 {
286  return __sync_fetch_and_xor(value, mask);
287 }
288 
289 #if defined(__x86_64__)
290 __INTRIN_INLINE long long _InterlockedXor64(volatile long long * value, long long mask)
291 {
292  return __sync_fetch_and_xor(value, mask);
293 }
294 #endif
295 
296 #if !HAS_BUILTIN(_InterlockedDecrement)
297 __INTRIN_INLINE long __cdecl _InterlockedDecrement(volatile long * lpAddend)
298 {
299  return __sync_sub_and_fetch(lpAddend, 1);
300 }
301 #endif
302 
303 #if !HAS_BUILTIN(_InterlockedIncrement)
304 __INTRIN_INLINE long __cdecl _InterlockedIncrement(volatile long * lpAddend)
305 {
306  return __sync_add_and_fetch(lpAddend, 1);
307 }
308 #endif
309 
310 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
311 {
312  return __sync_sub_and_fetch(lpAddend, 1);
313 }
314 
315 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
316 {
317  return __sync_add_and_fetch(lpAddend, 1);
318 }
319 
320 #if defined(__x86_64__)
321 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
322 {
323  return __sync_sub_and_fetch(lpAddend, 1);
324 }
325 
326 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
327 {
328  return __sync_add_and_fetch(lpAddend, 1);
329 }
330 #endif
331 
332 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
333 
334 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
335 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
336 {
337  char retval = Comperand;
338  __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
339  return retval;
340 }
341 #endif
342 
343 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
344 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
345 {
346  short retval = Comperand;
347  __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
348  return retval;
349 }
350 #endif
351 
352 #if !HAS_BUILTIN(_InterlockedCompareExchange)
353 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
354 {
355  long retval = Comperand;
356  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
357  return retval;
358 }
359 #endif
360 
361 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
362 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
363 {
364  void * retval = (void *)Comperand;
365  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
366  return retval;
367 }
368 #endif
369 
370 #if !HAS_BUILTIN(_InterlockedExchange8)
371 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
372 {
373  char retval = Value;
374  __asm__("xchgb %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
375  return retval;
376 }
377 #endif
378 
379 #if !HAS_BUILTIN(_InterlockedExchange16)
380 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
381 {
382  short retval = Value;
383  __asm__("xchgw %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
384  return retval;
385 }
386 #endif
387 
388 #if !HAS_BUILTIN(_InterlockedExchange)
389 __INTRIN_INLINE long _InterlockedExchange(volatile long * Target, long Value)
390 {
391  long retval = Value;
392  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
393  return retval;
394 }
395 #endif
396 
397 #if !HAS_BUILTIN(_InterlockedExchangePointer)
398 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
399 {
400  void * retval = Value;
401  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
402  return retval;
403 }
404 #endif
405 
406 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
407 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
408 {
409  char retval = Value;
410  __asm__("lock; xaddb %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
411  return retval;
412 }
413 #endif
414 
415 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
416 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
417 {
418  short retval = Value;
419  __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
420  return retval;
421 }
422 #endif
423 
424 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
425 __INTRIN_INLINE long _InterlockedExchangeAdd(volatile long * Addend, long Value)
426 {
427  long retval = Value;
428  __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
429  return retval;
430 }
431 #endif
432 
433 #if !HAS_BUILTIN(_InterlockedAnd8)
434 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
435 {
436  char x;
437  char y;
438 
439  y = *value;
440 
441  do
442  {
443  x = y;
444  y = _InterlockedCompareExchange8(value, x & mask, x);
445  }
446  while(y != x);
447 
448  return y;
449 }
450 #endif
451 
452 #if !HAS_BUILTIN(_InterlockedAnd16)
453 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
454 {
455  short x;
456  short y;
457 
458  y = *value;
459 
460  do
461  {
462  x = y;
463  y = _InterlockedCompareExchange16(value, x & mask, x);
464  }
465  while(y != x);
466 
467  return y;
468 }
469 #endif
470 
471 #if !HAS_BUILTIN(_InterlockedAnd)
472 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
473 {
474  long x;
475  long y;
476 
477  y = *value;
478 
479  do
480  {
481  x = y;
482  y = _InterlockedCompareExchange(value, x & mask, x);
483  }
484  while(y != x);
485 
486  return y;
487 }
488 #endif
489 
490 #if !HAS_BUILTIN(_InterlockedOr8)
491 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
492 {
493  char x;
494  char y;
495 
496  y = *value;
497 
498  do
499  {
500  x = y;
501  y = _InterlockedCompareExchange8(value, x | mask, x);
502  }
503  while(y != x);
504 
505  return y;
506 }
507 #endif
508 
509 #if !HAS_BUILTIN(_InterlockedOr16)
510 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
511 {
512  short x;
513  short y;
514 
515  y = *value;
516 
517  do
518  {
519  x = y;
520  y = _InterlockedCompareExchange16(value, x | mask, x);
521  }
522  while(y != x);
523 
524  return y;
525 }
526 #endif
527 
528 #if !HAS_BUILTIN(_InterlockedOr)
529 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
530 {
531  long x;
532  long y;
533 
534  y = *value;
535 
536  do
537  {
538  x = y;
539  y = _InterlockedCompareExchange(value, x | mask, x);
540  }
541  while(y != x);
542 
543  return y;
544 }
545 #endif
546 
547 #if !HAS_BUILTIN(_InterlockedXor8)
548 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
549 {
550  char x;
551  char y;
552 
553  y = *value;
554 
555  do
556  {
557  x = y;
558  y = _InterlockedCompareExchange8(value, x ^ mask, x);
559  }
560  while(y != x);
561 
562  return y;
563 }
564 #endif
565 
566 #if !HAS_BUILTIN(_InterlockedXor16)
567 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
568 {
569  short x;
570  short y;
571 
572  y = *value;
573 
574  do
575  {
576  x = y;
577  y = _InterlockedCompareExchange16(value, x ^ mask, x);
578  }
579  while(y != x);
580 
581  return y;
582 }
583 #endif
584 
585 #if !HAS_BUILTIN(_InterlockedXor)
586 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
587 {
588  long x;
589  long y;
590 
591  y = *value;
592 
593  do
594  {
595  x = y;
596  y = _InterlockedCompareExchange(value, x ^ mask, x);
597  }
598  while(y != x);
599 
600  return y;
601 }
602 #endif
603 
604 #if !HAS_BUILTIN(_InterlockedDecrement)
605 __INTRIN_INLINE long _InterlockedDecrement(volatile long * lpAddend)
606 {
607  return _InterlockedExchangeAdd(lpAddend, -1) - 1;
608 }
609 #endif
610 
611 #if !HAS_BUILTIN(_InterlockedIncrement)
612 __INTRIN_INLINE long _InterlockedIncrement(volatile long * lpAddend)
613 {
614  return _InterlockedExchangeAdd(lpAddend, 1) + 1;
615 }
616 #endif
617 
618 #if !HAS_BUILTIN(_InterlockedDecrement16)
619 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
620 {
621  return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
622 }
623 #endif
624 
625 #if !HAS_BUILTIN(_InterlockedIncrement16)
626 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
627 {
628  return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
629 }
630 #endif
631 
632 #if defined(__x86_64__)
633 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
634 {
635  return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
636 }
637 
638 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
639 {
640  return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
641 }
642 #endif
643 
644 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
645 
646 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
647 
648 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
649 {
650  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
651 }
652 
653 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
654 
655 #if !HAS_BUILTIN(_InterlockedCompareExchange64)
656 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
657 {
658  long long retval = Comperand;
659 
660  __asm__
661  (
662  "lock; cmpxchg8b %[Destination]" :
663  [retval] "+A" (retval) :
664  [Destination] "m" (*Destination),
665  "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
666  "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
667  "memory"
668  );
669 
670  return retval;
671 }
672 #endif
673 
674 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
675 
676 #ifdef __i386__
677 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * Addend, long Value)
678 {
679  __asm__
680  (
681  "lock; addl %[Value], %[Lo32];"
682  "jae LABEL%=;"
683  "lock; adcl $0, %[Hi32];"
684  "LABEL%=:;" :
685  [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
686  [Value] "ir" (Value) :
687  "memory"
688  );
689 
690  return Value;
691 }
692 #endif /* __i386__ */
693 
694 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, long b)
695 {
696  unsigned char retval;
697  __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
698  return retval;
699 }
700 
701 #if defined(__x86_64__)
702 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, long long b)
703 {
704  unsigned char retval;
705  __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
706  return retval;
707 }
708 #endif
709 
710 #if !HAS_BUILTIN(_interlockedbittestandset)
711 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, long b)
712 {
713  unsigned char retval;
714  __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
715  return retval;
716 }
717 #endif
718 
719 #if defined(__x86_64__)
720 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, long long b)
721 {
722  unsigned char retval;
723  __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
724  return retval;
725 }
726 #endif
727 
728 /*** String operations ***/
729 
730 #if !HAS_BUILTIN(__stosb)
731 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
732 __INTRIN_INLINE void __stosb(unsigned char * Dest, unsigned char Data, size_t Count)
733 {
734  __asm__ __volatile__
735  (
736  "rep; stosb" :
737  [Dest] "=D" (Dest), [Count] "=c" (Count) :
738  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
739  );
740 }
741 #endif
742 
743 __INTRIN_INLINE void __stosw(unsigned short * Dest, unsigned short Data, size_t Count)
744 {
745  __asm__ __volatile__
746  (
747  "rep; stosw" :
748  [Dest] "=D" (Dest), [Count] "=c" (Count) :
749  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
750  );
751 }
752 
753 __INTRIN_INLINE void __stosd(unsigned long * Dest, unsigned long Data, size_t Count)
754 {
755  __asm__ __volatile__
756  (
757  "rep; stosl" :
758  [Dest] "=D" (Dest), [Count] "=c" (Count) :
759  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
760  );
761 }
762 
763 #ifdef __x86_64__
764 __INTRIN_INLINE void __stosq(unsigned long long * Dest, unsigned long long Data, size_t Count)
765 {
766  __asm__ __volatile__
767  (
768  "rep; stosq" :
769  [Dest] "=D" (Dest), [Count] "=c" (Count) :
770  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
771  );
772 }
773 #endif
774 
775 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
776 {
777  __asm__ __volatile__
778  (
779  "rep; movsb" :
780  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
781  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
782  );
783 }
784 
785 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
786 {
787  __asm__ __volatile__
788  (
789  "rep; movsw" :
790  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
791  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
792  );
793 }
794 
795 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
796 {
797  __asm__ __volatile__
798  (
799  "rep; movsd" :
800  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
801  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
802  );
803 }
804 
805 #ifdef __x86_64__
806 __INTRIN_INLINE void __movsq(unsigned long * Destination, const unsigned long * Source, size_t Count)
807 {
808  __asm__ __volatile__
809  (
810  "rep; movsq" :
811  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
812  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
813  );
814 }
815 #endif
816 
817 #if defined(__x86_64__)
818 
819 /*** GS segment addressing ***/
820 
821 __INTRIN_INLINE void __writegsbyte(unsigned long Offset, unsigned char Data)
822 {
823  __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
824 }
825 
826 __INTRIN_INLINE void __writegsword(unsigned long Offset, unsigned short Data)
827 {
828  __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
829 }
830 
831 __INTRIN_INLINE void __writegsdword(unsigned long Offset, unsigned long Data)
832 {
833  __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
834 }
835 
836 __INTRIN_INLINE void __writegsqword(unsigned long Offset, unsigned long long Data)
837 {
838  __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
839 }
840 
841 __INTRIN_INLINE unsigned char __readgsbyte(unsigned long Offset)
842 {
843  unsigned char value;
844  __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
845  return value;
846 }
847 
848 __INTRIN_INLINE unsigned short __readgsword(unsigned long Offset)
849 {
850  unsigned short value;
851  __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
852  return value;
853 }
854 
855 __INTRIN_INLINE unsigned long __readgsdword(unsigned long Offset)
856 {
857  unsigned long value;
858  __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
859  return value;
860 }
861 
862 __INTRIN_INLINE unsigned long long __readgsqword(unsigned long Offset)
863 {
864  unsigned long long value;
865  __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
866  return value;
867 }
868 
869 __INTRIN_INLINE void __incgsbyte(unsigned long Offset)
870 {
871  __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
872 }
873 
874 __INTRIN_INLINE void __incgsword(unsigned long Offset)
875 {
876  __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
877 }
878 
879 __INTRIN_INLINE void __incgsdword(unsigned long Offset)
880 {
881  __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
882 }
883 
884 __INTRIN_INLINE void __incgsqword(unsigned long Offset)
885 {
886  __asm__ __volatile__("incq %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
887 }
888 
889 __INTRIN_INLINE void __addgsbyte(unsigned long Offset, unsigned char Data)
890 {
891  __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
892 }
893 
894 __INTRIN_INLINE void __addgsword(unsigned long Offset, unsigned short Data)
895 {
896  __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
897 }
898 
899 __INTRIN_INLINE void __addgsdword(unsigned long Offset, unsigned int Data)
900 {
901  __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
902 }
903 
904 __INTRIN_INLINE void __addgsqword(unsigned long Offset, unsigned long long Data)
905 {
906  __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
907 }
908 
909 #else /* defined(__x86_64__) */
910 
911 /*** FS segment addressing ***/
912 
913 __INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
914 {
915  __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
916 }
917 
918 __INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
919 {
920  __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
921 }
922 
923 __INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
924 {
925  __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
926 }
927 
928 #if !HAS_BUILTIN(__readfsbyte)
929 __INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
930 {
931  unsigned char value;
932  __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
933  return value;
934 }
935 #endif
936 
937 #if !HAS_BUILTIN(__readfsword)
938 __INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
939 {
940  unsigned short value;
941  __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
942  return value;
943 }
944 #endif
945 
946 #if !HAS_BUILTIN(__readfsdword)
947 __INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
948 {
949  unsigned long value;
950  __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
951  return value;
952 }
953 #endif
954 
955 __INTRIN_INLINE void __incfsbyte(unsigned long Offset)
956 {
957  __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
958 }
959 
960 __INTRIN_INLINE void __incfsword(unsigned long Offset)
961 {
962  __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
963 }
964 
965 __INTRIN_INLINE void __incfsdword(unsigned long Offset)
966 {
967  __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
968 }
969 
970 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
971 __INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
972 {
973  if(!__builtin_constant_p(Offset))
974  __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
975  else
976  __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
977 }
978 
979 __INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
980 {
981  if(!__builtin_constant_p(Offset))
982  __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
983  else
984  __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
985 }
986 
987 __INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
988 {
989  if(!__builtin_constant_p(Offset))
990  __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
991  else
992  __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
993 }
994 
995 #endif /* defined(__x86_64__) */
996 
997 
998 /*** Bit manipulation ***/
999 
1000 #if !HAS_BUILTIN(_BitScanForward)
1001 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * Index, unsigned long Mask)
1002 {
1003  __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1004  return Mask ? 1 : 0;
1005 }
1006 #endif
1007 
1008 #if !HAS_BUILTIN(_BitScanReverse)
1009 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * Index, unsigned long Mask)
1010 {
1011  __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1012  return Mask ? 1 : 0;
1013 }
1014 #endif
1015 
1016 /* NOTE: again, the bizarre implementation follows Visual C++ */
1017 __INTRIN_INLINE unsigned char _bittest(const long * a, long b)
1018 {
1019  unsigned char retval;
1020 
1021  if(__builtin_constant_p(b))
1022  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
1023  else
1024  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1025 
1026  return retval;
1027 }
1028 
1029 #ifdef __x86_64__
1030 __INTRIN_INLINE unsigned char _BitScanForward64(unsigned long * Index, unsigned long long Mask)
1031 {
1032  unsigned long long Index64;
1033  __asm__("bsfq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1034  *Index = Index64;
1035  return Mask ? 1 : 0;
1036 }
1037 
1038 __INTRIN_INLINE unsigned char _BitScanReverse64(unsigned long * Index, unsigned long long Mask)
1039 {
1040  unsigned long long Index64;
1041  __asm__("bsrq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1042  *Index = Index64;
1043  return Mask ? 1 : 0;
1044 }
1045 
1046 __INTRIN_INLINE unsigned char _bittest64(const long long * a, long long b)
1047 {
1048  unsigned char retval;
1049 
1050  if(__builtin_constant_p(b))
1051  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
1052  else
1053  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1054 
1055  return retval;
1056 }
1057 #endif
1058 
1059 __INTRIN_INLINE unsigned char _bittestandcomplement(long * a, long b)
1060 {
1061  unsigned char retval;
1062 
1063  if(__builtin_constant_p(b))
1064  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1065  else
1066  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1067 
1068  return retval;
1069 }
1070 
1071 __INTRIN_INLINE unsigned char _bittestandreset(long * a, long b)
1072 {
1073  unsigned char retval;
1074 
1075  if(__builtin_constant_p(b))
1076  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1077  else
1078  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1079 
1080  return retval;
1081 }
1082 
1083 __INTRIN_INLINE unsigned char _bittestandset(long * a, long b)
1084 {
1085  unsigned char retval;
1086 
1087  if(__builtin_constant_p(b))
1088  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1089  else
1090  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1091 
1092  return retval;
1093 }
1094 
1095 #ifdef __x86_64__
1096 
1097 __INTRIN_INLINE unsigned char _bittestandset64(long long * a, long long b)
1098 {
1099  unsigned char retval;
1100 
1101  if(__builtin_constant_p(b))
1102  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1103  else
1104  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1105 
1106  return retval;
1107 }
1108 
1109 __INTRIN_INLINE unsigned char _bittestandreset64(long long * a, long long b)
1110 {
1111  unsigned char retval;
1112 
1113  if(__builtin_constant_p(b))
1114  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1115  else
1116  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1117 
1118  return retval;
1119 }
1120 
1121 __INTRIN_INLINE unsigned char _bittestandcomplement64(long long * a, long long b)
1122 {
1123  unsigned char retval;
1124 
1125  if(__builtin_constant_p(b))
1126  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1127  else
1128  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1129 
1130  return retval;
1131 }
1132 
1133 #endif /* __x86_64__ */
1134 
1135 #if !HAS_BUILTIN(_rotl8)
1136 __INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
1137 {
1138  unsigned char retval;
1139  __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1140  return retval;
1141 }
1142 #endif
1143 
1144 #if !HAS_BUILTIN(_rotl16)
1145 __INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
1146 {
1147  unsigned short retval;
1148  __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1149  return retval;
1150 }
1151 #endif
1152 
1153 #if !HAS_BUILTIN(_rotl)
1154 __INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
1155 {
1156  unsigned int retval;
1157  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1158  return retval;
1159 }
1160 #endif
1161 
1162 #ifdef __x86_64__
1163 __INTRIN_INLINE unsigned long long _rotl64(unsigned long long value, int shift)
1164 {
1165  unsigned long long retval;
1166  __asm__("rolq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1167  return retval;
1168 }
1169 #else /* __x86_64__ */
1170 #if !HAS_BUILTIN(_rotl64)
1171 __INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
1172 {
1173  /* FIXME: this is probably not optimal */
1174  return (value << shift) | (value >> (64 - shift));
1175 }
1176 #endif /* !HAS_BUILTIN(_rotl64) */
1177 #endif /* __x86_64__ */
1178 
1179 #if !HAS_BUILTIN(_rotr)
1180 __INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
1181 {
1182  unsigned int retval;
1183  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1184  return retval;
1185 }
1186 #endif
1187 
1188 #if !HAS_BUILTIN(_rotr8)
1189 __INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
1190 {
1191  unsigned char retval;
1192  __asm__("rorb %b[shift], %b[retval]" : [retval] "=qm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1193  return retval;
1194 }
1195 #endif
1196 
1197 #if !HAS_BUILTIN(_rotr16)
1198 __INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
1199 {
1200  unsigned short retval;
1201  __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1202  return retval;
1203 }
1204 #endif
1205 
1206 #ifdef __x86_64__
1207 __INTRIN_INLINE unsigned long long _rotr64(unsigned long long value, int shift)
1208 {
1209  unsigned long long retval;
1210  __asm__("rorq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1211  return retval;
1212 }
1213 #else /* __x86_64__ */
1214 #if !HAS_BUILTIN(_rotr64)
1215 __INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
1216 {
1217  /* FIXME: this is probably not optimal */
1218  return (value >> shift) | (value << (64 - shift));
1219 }
1220 #endif /* !HAS_BUILTIN(_rotr64) */
1221 #endif /* __x86_64__ */
1222 
1223 #if !HAS_BUILTIN(_lrotl)
1224 __INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
1225 {
1226  unsigned long retval;
1227  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1228  return retval;
1229 }
1230 #endif
1231 
1232 #if !HAS_BUILTIN(_lrotr)
1233 __INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
1234 {
1235  unsigned long retval;
1236  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1237  return retval;
1238 }
1239 #endif
1240 
1241 /*
1242  NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1243  constraint (edx:eax) for the Mask argument, because it's the only way GCC
1244  can pass 64-bit operands around - passing the two 32 bit parts separately
1245  just confuses it. Also we declare Bit as an int and then truncate it to
1246  match Visual C++ behavior
1247 */
1248 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1249 {
1250  unsigned long long retval = Mask;
1251 
1252  __asm__
1253  (
1254  "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1255  "+A" (retval) :
1256  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1257  );
1258 
1259  return retval;
1260 }
1261 
1262 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1263 {
1264  long long retval = Mask;
1265 
1266  __asm__
1267  (
1268  "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1269  "+A" (retval) :
1270  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1271  );
1272 
1273  return retval;
1274 }
1275 
1276 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1277 {
1278  unsigned long long retval = Mask;
1279 
1280  __asm__
1281  (
1282  "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1283  "+A" (retval) :
1284  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1285  );
1286 
1287  return retval;
1288 }
1289 
1290 __INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
1291 {
1292  unsigned short retval;
1293  __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
1294  return retval;
1295 }
1296 
1297 __INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
1298 {
1299  unsigned long retval;
1300  __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1301  return retval;
1302 }
1303 
1304 #ifdef __x86_64__
1305 __INTRIN_INLINE unsigned long long _byteswap_uint64(unsigned long long value)
1306 {
1307  unsigned long long retval;
1308  __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1309  return retval;
1310 }
1311 #else
1312 __INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
1313 {
1314  union {
1315  unsigned long long int64part;
1316  struct {
1317  unsigned long lowpart;
1318  unsigned long hipart;
1319  };
1320  } retval;
1321  retval.int64part = value;
1322  __asm__("bswapl %[lowpart]\n"
1323  "bswapl %[hipart]\n"
1324  : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
1325  return retval.int64part;
1326 }
1327 #endif
1328 
1329 __INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
1330 {
1331  return __builtin_clz(value);
1332 }
1333 
1334 __INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
1335 {
1336  return __builtin_clz(value);
1337 }
1338 
1339 #if !HAS_BUILTIN(__popcnt)
1340 __INTRIN_INLINE unsigned int __popcnt(unsigned int value)
1341 {
1342  return __builtin_popcount(value);
1343 }
1344 #endif
1345 
1346 #if !HAS_BUILTIN(__popcnt16)
1347 __INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
1348 {
1349  return __builtin_popcount(value);
1350 }
1351 #endif
1352 
1353 #ifdef __x86_64__
1354 __INTRIN_INLINE unsigned long long __lzcnt64(unsigned long long value)
1355 {
1356  return __builtin_clzll(value);
1357 }
1358 
1359 __INTRIN_INLINE unsigned long long __popcnt64(unsigned long long value)
1360 {
1361  return __builtin_popcountll(value);
1362 }
1363 #endif
1364 
1365 /*** 64-bit math ***/
1366 
1367 #if !HAS_BUILTIN(__emul)
1368 __INTRIN_INLINE long long __emul(int a, int b)
1369 {
1370  long long retval;
1371  __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1372  return retval;
1373 }
1374 #endif
1375 
1376 #if !HAS_BUILTIN(__emulu)
1377 __INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
1378 {
1379  unsigned long long retval;
1380  __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1381  return retval;
1382 }
1383 #endif
1384 
1385 __INTRIN_INLINE long long __cdecl _abs64(long long value)
1386 {
1387  return (value >= 0) ? value : -value;
1388 }
1389 
1390 #ifdef __x86_64__
1391 
1392 __INTRIN_INLINE long long __mulh(long long a, long long b)
1393 {
1394  long long retval;
1395  __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1396  return retval;
1397 }
1398 
1399 __INTRIN_INLINE unsigned long long __umulh(unsigned long long a, unsigned long long b)
1400 {
1401  unsigned long long retval;
1402  __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1403  return retval;
1404 }
1405 
1406 #endif
1407 
1408 /*** Port I/O ***/
1409 
1410 __INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
1411 {
1412  unsigned char byte;
1413  __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1414  return byte;
1415 }
1416 
1417 __INTRIN_INLINE unsigned short __inword(unsigned short Port)
1418 {
1419  unsigned short word;
1420  __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1421  return word;
1422 }
1423 
1424 __INTRIN_INLINE unsigned long __indword(unsigned short Port)
1425 {
1426  unsigned long dword;
1427  __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1428  return dword;
1429 }
1430 
1431 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1432 {
1433  __asm__ __volatile__
1434  (
1435  "rep; insb" :
1436  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1437  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1438  "memory"
1439  );
1440 }
1441 
1442 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1443 {
1444  __asm__ __volatile__
1445  (
1446  "rep; insw" :
1447  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1448  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1449  "memory"
1450  );
1451 }
1452 
1453 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1454 {
1455  __asm__ __volatile__
1456  (
1457  "rep; insl" :
1458  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1459  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1460  "memory"
1461  );
1462 }
1463 
1464 __INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
1465 {
1466  __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1467 }
1468 
1469 __INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
1470 {
1471  __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1472 }
1473 
1474 __INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
1475 {
1476  __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1477 }
1478 
1479 __INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1480 {
1481  __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1482 }
1483 
1484 __INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1485 {
1486  __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1487 }
1488 
1489 __INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1490 {
1491  __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1492 }
1493 
1494 __INTRIN_INLINE int __cdecl _inp(unsigned short Port)
1495 {
1496  return __inbyte(Port);
1497 }
1498 
1499 __INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
1500 {
1501  return __inword(Port);
1502 }
1503 
1504 __INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
1505 {
1506  return __indword(Port);
1507 }
1508 
1509 __INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
1510 {
1511  __outbyte(Port, (unsigned char)databyte);
1512  return databyte;
1513 }
1514 
1515 __INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
1516 {
1517  __outword(Port, dataword);
1518  return dataword;
1519 }
1520 
1521 __INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
1522 {
1523  __outdword(Port, dataword);
1524  return dataword;
1525 }
1526 
1527 
1528 /*** System information ***/
1529 
1530 __INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
1531 {
1532  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1533 }
1534 
1535 __INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
1536 {
1537  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType), "c" (ECXValue));
1538 }
1539 
1540 #if !HAS_BUILTIN(__rdtsc)
1541 __INTRIN_INLINE unsigned long long __rdtsc(void)
1542 {
1543 #ifdef __x86_64__
1544  unsigned long long low, high;
1545  __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1546  return low | (high << 32);
1547 #else
1548  unsigned long long retval;
1549  __asm__ __volatile__("rdtsc" : "=A"(retval));
1550  return retval;
1551 #endif
1552 }
1553 #endif /* !HAS_BUILTIN(__rdtsc) */
1554 
1556 {
1557  __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1558 }
1559 
1561 {
1562  uintptr_t retval;
1563  __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1564  return retval;
1565 }
1566 
1567 /*** Interrupts ***/
1568 
1569 #if !HAS_BUILTIN(__debugbreak)
1571 {
1572  __asm__("int $3");
1573 }
1574 #endif
1575 
1576 #if !HAS_BUILTIN(__ud2)
1578 {
1579  __asm__("ud2");
1580 }
1581 #endif
1582 
1583 #if !HAS_BUILTIN(__int2c)
1585 {
1586  __asm__("int $0x2c");
1587 }
1588 #endif
1589 
1591 {
1592  __asm__("cli" : : : "memory");
1593 }
1594 
1596 {
1597  __asm__("sti" : : : "memory");
1598 }
1599 
1601 {
1602  __asm__("hlt" : : : "memory");
1603 }
1604 
1605 #if !HAS_BUILTIN(__fastfail)
1606 __declspec(noreturn)
1607 __INTRIN_INLINE void __fastfail(unsigned int Code)
1608 {
1609  __asm__("int $0x29" : : "c"(Code) : "memory");
1610  __builtin_unreachable();
1611 }
1612 #endif
1613 
1614 /*** Protected memory management ***/
1615 
1616 #ifdef __x86_64__
1617 
1618 __INTRIN_INLINE void __writecr0(unsigned long long Data)
1619 {
1620  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1621 }
1622 
1623 __INTRIN_INLINE void __writecr3(unsigned long long Data)
1624 {
1625  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1626 }
1627 
1628 __INTRIN_INLINE void __writecr4(unsigned long long Data)
1629 {
1630  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1631 }
1632 
1633 __INTRIN_INLINE void __writecr8(unsigned long long Data)
1634 {
1635  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1636 }
1637 
1638 __INTRIN_INLINE unsigned long long __readcr0(void)
1639 {
1640  unsigned long long value;
1641  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1642  return value;
1643 }
1644 
1645 __INTRIN_INLINE unsigned long long __readcr2(void)
1646 {
1647  unsigned long long value;
1648  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1649  return value;
1650 }
1651 
1652 __INTRIN_INLINE unsigned long long __readcr3(void)
1653 {
1654  unsigned long long value;
1655  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1656  return value;
1657 }
1658 
1659 __INTRIN_INLINE unsigned long long __readcr4(void)
1660 {
1661  unsigned long long value;
1662  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1663  return value;
1664 }
1665 
1666 __INTRIN_INLINE unsigned long long __readcr8(void)
1667 {
1668  unsigned long long value;
1669  __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1670  return value;
1671 }
1672 
1673 #else /* __x86_64__ */
1674 
1675 __INTRIN_INLINE void __writecr0(unsigned int Data)
1676 {
1677  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1678 }
1679 
1680 __INTRIN_INLINE void __writecr3(unsigned int Data)
1681 {
1682  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1683 }
1684 
1685 __INTRIN_INLINE void __writecr4(unsigned int Data)
1686 {
1687  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1688 }
1689 
1690 #if !HAS_BUILTIN(__writecr8)
1691 __INTRIN_INLINE void __writecr8(unsigned int Data)
1692 {
1693  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1694 }
1695 #endif
1696 
1697 __INTRIN_INLINE unsigned long __readcr0(void)
1698 {
1699  unsigned long value;
1700  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1701  return value;
1702 }
1703 
1704 __INTRIN_INLINE unsigned long __readcr2(void)
1705 {
1706  unsigned long value;
1707  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1708  return value;
1709 }
1710 
1711 __INTRIN_INLINE unsigned long __readcr3(void)
1712 {
1713  unsigned long value;
1714  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1715  return value;
1716 }
1717 
1718 __INTRIN_INLINE unsigned long __readcr4(void)
1719 {
1720  unsigned long value;
1721  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1722  return value;
1723 }
1724 
1725 #if !HAS_BUILTIN(__readcr8)
1726 __INTRIN_INLINE unsigned long __readcr8(void)
1727 {
1728  unsigned long value;
1729  __asm__ __volatile__("mov %%cr8, %[value]" : [value] "=r" (value));
1730  return value;
1731 }
1732 #endif
1733 
1734 #endif /* __x86_64__ */
1735 
1736 #ifdef __x86_64__
1737 
1738 __INTRIN_INLINE unsigned long long __readdr(unsigned int reg)
1739 {
1740  unsigned long long value;
1741  switch (reg)
1742  {
1743  case 0:
1744  __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1745  break;
1746  case 1:
1747  __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1748  break;
1749  case 2:
1750  __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1751  break;
1752  case 3:
1753  __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1754  break;
1755  case 4:
1756  __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1757  break;
1758  case 5:
1759  __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1760  break;
1761  case 6:
1762  __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1763  break;
1764  case 7:
1765  __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1766  break;
1767  }
1768  return value;
1769 }
1770 
1771 __INTRIN_INLINE void __writedr(unsigned reg, unsigned long long value)
1772 {
1773  switch (reg)
1774  {
1775  case 0:
1776  __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1777  break;
1778  case 1:
1779  __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1780  break;
1781  case 2:
1782  __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1783  break;
1784  case 3:
1785  __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1786  break;
1787  case 4:
1788  __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1789  break;
1790  case 5:
1791  __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1792  break;
1793  case 6:
1794  __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1795  break;
1796  case 7:
1797  __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1798  break;
1799  }
1800 }
1801 
1802 #else /* __x86_64__ */
1803 
1804 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1805 {
1806  unsigned int value;
1807  switch (reg)
1808  {
1809  case 0:
1810  __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1811  break;
1812  case 1:
1813  __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1814  break;
1815  case 2:
1816  __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1817  break;
1818  case 3:
1819  __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1820  break;
1821  case 4:
1822  __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1823  break;
1824  case 5:
1825  __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1826  break;
1827  case 6:
1828  __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1829  break;
1830  case 7:
1831  __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1832  break;
1833  }
1834  return value;
1835 }
1836 
1837 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1838 {
1839  switch (reg)
1840  {
1841  case 0:
1842  __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1843  break;
1844  case 1:
1845  __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1846  break;
1847  case 2:
1848  __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1849  break;
1850  case 3:
1851  __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1852  break;
1853  case 4:
1854  __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1855  break;
1856  case 5:
1857  __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1858  break;
1859  case 6:
1860  __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1861  break;
1862  case 7:
1863  __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1864  break;
1865  }
1866 }
1867 
1868 #endif /* __x86_64__ */
1869 
1871 {
1872  __asm__ __volatile__ ("invlpg (%[Address])" : : [Address] "b" (Address) : "memory");
1873 }
1874 
1875 
1876 /*** System operations ***/
1877 
1878 __INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
1879 {
1880 #ifdef __x86_64__
1881  unsigned long low, high;
1882  __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1883  return ((unsigned long long)high << 32) | low;
1884 #else
1885  unsigned long long retval;
1886  __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1887  return retval;
1888 #endif
1889 }
1890 
1891 __INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
1892 {
1893 #ifdef __x86_64__
1894  __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1895 #else
1896  __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1897 #endif
1898 }
1899 
1900 __INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
1901 {
1902  unsigned long long retval;
1903  __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
1904  return retval;
1905 }
1906 
1907 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1908 __INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
1909 {
1910  unsigned long retval;
1911  __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
1912  return retval;
1913 }
1914 
1916 {
1917  __asm__ __volatile__("wbinvd" : : : "memory");
1918 }
1919 
1920 __INTRIN_INLINE void __lidt(void *Source)
1921 {
1922  __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
1923 }
1924 
1925 __INTRIN_INLINE void __sidt(void *Destination)
1926 {
1927  __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
1928 }
1929 
1930 __INTRIN_INLINE void _sgdt(void *Destination)
1931 {
1932  __asm__ __volatile__("sgdt %0" : : "m"(*(short*)Destination) : "memory");
1933 }
1934 
1935 /*** Misc operations ***/
1936 
1937 #if !HAS_BUILTIN(_mm_pause)
1939 {
1940  __asm__ __volatile__("pause" : : : "memory");
1941 }
1942 #endif
1943 
1945 {
1946  __asm__ __volatile__("nop");
1947 }
1948 
1949 #ifdef __cplusplus
1950 }
1951 #endif
1952 
1953 #endif /* KJK_INTRIN_X86_H_ */
1954 
1955 /* EOF */
__INTRIN_INLINE void __ud2(void)
Definition: intrin_x86.h:1577
CPPORT Port[4]
Definition: headless.c:34
__INTRIN_INLINE void __writecr4(unsigned int Data)
Definition: intrin_x86.h:1685
__INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
Definition: intrin_x86.h:1499
__INTRIN_INLINE void __stosd(unsigned long *Dest, unsigned long Data, size_t Count)
Definition: intrin_x86.h:753
__INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
Definition: intrin_x86.h:1837
Definition: get.c:139
__INTRIN_INLINE void _mm_lfence(void)
Definition: intrin_x86.h:111
__INTRIN_INLINE void __stosb(unsigned char *Dest, unsigned char Data, size_t Count)
Definition: intrin_x86.h:732
__INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
Definition: intrin_x86.h:1171
PVOID ULONG Address
Definition: oprghdlr.h:14
#define shift
Definition: input.c:1668
__INTRIN_INLINE unsigned long __readcr8(void)
Definition: intrin_x86.h:1726
__INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:971
__INTRIN_INLINE void *__cdecl memcpy(void *dest, const void *source, size_t num)
Definition: intrin_x86.h:74
__INTRIN_INLINE char _InterlockedCompareExchange8(volatile char *Destination, char Exchange, char Comperand)
Definition: intrin_x86.h:335
__INTRIN_INLINE unsigned long __readcr2(void)
Definition: intrin_x86.h:1704
__INTRIN_INLINE void __movsb(unsigned char *Destination, const unsigned char *Source, size_t Count)
Definition: intrin_x86.h:775
__INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
Definition: intrin_x86.h:1878
__INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long *a, long b)
Definition: intrin_x86.h:694
#define __cdecl
Definition: accygwin.h:79
__INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:913
__INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1145
__INTRIN_INLINE void __incfsdword(unsigned long Offset)
Definition: intrin_x86.h:965
__INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
Definition: intrin_x86.h:947
__INTRIN_INLINE unsigned char _BitScanReverse(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1009
__INTRIN_INLINE void __wbinvd(void)
Definition: intrin_x86.h:1915
__INTRIN_INLINE long _InterlockedXor(volatile long *value, long mask)
Definition: intrin_x86.h:586
__INTRIN_INLINE void __incfsword(unsigned long Offset)
Definition: intrin_x86.h:960
__INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1431
__INTRIN_INLINE void __halt(void)
Definition: intrin_x86.h:1600
__INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long *Destination, long long Exchange, long long Comperand)
Definition: intrin_x86.h:656
__INTRIN_INLINE void __stosw(unsigned short *Dest, unsigned short Data, size_t Count)
Definition: intrin_x86.h:743
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
__INTRIN_INLINE void __lidt(void *Source)
Definition: intrin_x86.h:1920
__INTRIN_INLINE unsigned int __popcnt(unsigned int value)
Definition: intrin_x86.h:1340
__INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1276
__INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
Definition: intrin_x86.h:1908
__declspec(noreturn) __INTRIN_INLINE void __fastfail(unsigned int Code)
Definition: intrin_x86.h:1606
__INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
Definition: intrin_x86.h:1312
__INTRIN_INLINE unsigned long __readcr3(void)
Definition: intrin_x86.h:1711
__INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
Definition: intrin_x86.h:929
__INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1453
__INTRIN_INLINE void __movsd(unsigned long *Destination, const unsigned long *Source, size_t Count)
Definition: intrin_x86.h:795
__INTRIN_INLINE long long __emul(int a, int b)
Definition: intrin_x86.h:1368
__asm__("\t.globl GetPhys\n""GetPhys:\t\n""mflr 0\n\t""stwu 0,-16(1)\n\t""mfmsr 5\n\t""andi. 6,5,0xffef\n\t""mtmsr 6\n\t""isync\n\t""sync\n\t""lwz 3,0(3)\n\t""mtmsr 5\n\t""isync\n\t""sync\n\t""lwz 0,0(1)\n\t""addi 1,1,16\n\t""mtlr 0\n\t""blr")
IN OUT PLONG Addend
Definition: CrNtStubs.h:22
__INTRIN_INLINE void __writecr8(unsigned int Data)
Definition: intrin_x86.h:1691
__INTRIN_INLINE short _InterlockedIncrement16(volatile short *lpAddend)
Definition: intrin_x86.h:626
__INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:918
__INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
Definition: intrin_x86.h:1154
__INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
Definition: intrin_x86.h:1215
__INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
Definition: intrin_x86.h:1329
__INTRIN_INLINE unsigned char _BitScanForward(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1001
__INTRIN_INLINE unsigned char _bittestandset(long *a, long b)
Definition: intrin_x86.h:1083
__INTRIN_INLINE long _InterlockedOr(volatile long *value, long mask)
Definition: intrin_x86.h:529
__INTRIN_INLINE int __cdecl _inp(unsigned short Port)
Definition: intrin_x86.h:1494
__INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
Definition: intrin_x86.h:1297
__INTRIN_INLINE void __cdecl __debugbreak(void)
Definition: intrin_x86.h:1570
__INTRIN_INLINE void __writecr3(unsigned int Data)
Definition: intrin_x86.h:1680
__INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1479
__INTRIN_INLINE unsigned char _bittestandreset(long *a, long b)
Definition: intrin_x86.h:1071
GLenum GLint GLuint mask
Definition: glext.h:6028
__INTRIN_INLINE uintptr_t __readeflags(void)
Definition: intrin_x86.h:1560
_In_ ULONG _In_ ULONG Offset
Definition: ntddpcm.h:101
__INTRIN_INLINE long _InterlockedIncrement(volatile long *lpAddend)
Definition: intrin_x86.h:612
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:1925
__INTRIN_INLINE void _ReadWriteBarrier(void)
Definition: intrin_x86.h:93
#define _ReadBarrier
Definition: intrin_x86.h:100
__INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:979
unsigned int uintptr_t
Definition: crtdefs.h:300
__INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:923
__INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long *const Addend, const long Value)
Definition: intrin_arm.h:359
__INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
Definition: intrin_x86.h:938
__INTRIN_INLINE void __writecr0(unsigned int Data)
Definition: intrin_x86.h:1675
__INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
Definition: intrin_x86.h:1410
__INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile *Addend, char Value)
Definition: intrin_x86.h:407
__INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1189
__INTRIN_INLINE short _InterlockedDecrement16(volatile short *lpAddend)
Definition: intrin_x86.h:619
Definition: bufpool.h:45
__INTRIN_INLINE long _InterlockedCompareExchange(volatile long *Destination, long Exchange, long Comperand)
Definition: intrin_x86.h:353
#define _WriteBarrier
Definition: intrin_x86.h:101
#define Code
Definition: deflate.h:80
__INTRIN_INLINE void __nop(void)
Definition: intrin_x86.h:1944
__INTRIN_INLINE long _InterlockedAnd(volatile long *value, long mask)
Definition: intrin_x86.h:472
__INTRIN_INLINE short _InterlockedAnd16(volatile short *value, short mask)
Definition: intrin_x86.h:453
__INTRIN_INLINE void _mm_pause(void)
Definition: intrin_x86.h:1938
GLboolean GLboolean GLboolean b
Definition: glext.h:6204
__INTRIN_INLINE long long __cdecl _abs64(long long value)
Definition: intrin_x86.h:1385
__INTRIN_INLINE void _mm_sfence(void)
Definition: intrin_x86.h:120
__INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
Definition: intrin_x86.h:1262
__INTRIN_INLINE char _InterlockedXor8(volatile char *value, char mask)
Definition: intrin_x86.h:548
__INTRIN_INLINE void * _InterlockedCompareExchangePointer(void *volatile *Destination, void *Exchange, void *Comperand)
Definition: intrin_x86.h:362
__INTRIN_INLINE long _InterlockedDecrement(volatile long *lpAddend)
Definition: intrin_x86.h:605
__INTRIN_INLINE void * _InterlockedExchangePointer(void *volatile *Target, void *Value)
Definition: intrin_x86.h:398
__INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
Definition: intrin_x86.h:1180
GLfloat CONST GLvector4f CONST GLfloat GLvector4f * dest
Definition: m_xform.h:122
static const UCHAR Index[8]
Definition: usbohci.c:18
__INTRIN_INLINE void __cdecl _enable(void)
Definition: intrin_x86.h:1595
__INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
Definition: intrin_x86.h:1891
GLuint GLuint num
Definition: glext.h:9618
__INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
Definition: intrin_x86.h:1530
_In_ PUNICODE_STRING _Inout_ PUNICODE_STRING Destination
Definition: rtlfuncs.h:2875
__INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
Definition: intrin_x86.h:1504
_Must_inspect_result_ typedef _In_ ULONG _In_ BOOLEAN Target
Definition: iotypes.h:1067
__INTRIN_INLINE char _InterlockedExchange8(volatile char *Target, char Value)
Definition: intrin_x86.h:371
__INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1489
__INTRIN_INLINE long _InterlockedExchangeAdd(volatile long *Addend, long Value)
Definition: intrin_x86.h:425
__INTRIN_INLINE void __int2c(void)
Definition: intrin_x86.h:1584
__INTRIN_INLINE long _InterlockedExchange(volatile long *Target, long Value)
Definition: intrin_x86.h:389
__INTRIN_INLINE short _InterlockedCompareExchange16(volatile short *Destination, short Exchange, short Comperand)
Definition: intrin_x86.h:344
#define __INTRIN_INLINE
Definition: intrin.h:39
__INTRIN_INLINE void __writeeflags(uintptr_t Value)
Definition: intrin_x86.h:1555
__INTRIN_INLINE unsigned long long __rdtsc(void)
Definition: intrin_x86.h:1541
__INTRIN_INLINE void __cdecl _disable(void)
Definition: intrin_x86.h:1590
__INTRIN_INLINE unsigned int __readdr(unsigned int reg)
Definition: intrin_x86.h:1804
__INTRIN_INLINE char _InterlockedOr8(volatile char *value, char mask)
Definition: intrin_x86.h:491
__INTRIN_INLINE unsigned long __readcr0(void)
Definition: intrin_x86.h:1697
__INTRIN_INLINE unsigned long __readcr4(void)
Definition: intrin_x86.h:1718
GLsizei const GLfloat * value
Definition: glext.h:6069
UINTN UINT8 Value
Definition: acefiex.h:751
__INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1442
__INTRIN_INLINE void _sgdt(void *Destination)
Definition: intrin_x86.h:1930
__INTRIN_INLINE unsigned long __indword(unsigned short Port)
Definition: intrin_x86.h:1424
__INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
Definition: intrin_x86.h:1377
#define local
Definition: zutil.h:30
__INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
Definition: intrin_x86.h:1900
__INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
Definition: intrin_x86.h:1334
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1535
__INTRIN_INLINE char _InterlockedAnd8(volatile char *value, char mask)
Definition: intrin_x86.h:434
#define byte(x, n)
Definition: tomcrypt.h:118
__INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:987
CPUINFO CPUInfo[]
Definition: parse.c:231
__INTRIN_INLINE void __incfsbyte(unsigned long Offset)
Definition: intrin_x86.h:955
__INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
Definition: intrin_x86.h:1515
__INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1248
static int reg
Definition: i386-dis.c:1275
void *__cdecl memmove(void *dest, const void *source, size_t num)
Definition: memmove.c:4
__INTRIN_INLINE short _InterlockedOr16(volatile short *value, short mask)
Definition: intrin_x86.h:510
const WCHAR * word
Definition: lex.c:70
__INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
Definition: intrin_x86.h:1509
__INTRIN_INLINE short _InterlockedExchange16(volatile short *Target, short Value)
Definition: intrin_x86.h:380
__INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short *Addend, short Value)
Definition: intrin_x86.h:416
__INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
Definition: intrin_x86.h:1233
__INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1136
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1870
__INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
Definition: intrin_x86.h:1224
__INTRIN_INLINE unsigned char _bittest(const long *a, long b)
Definition: intrin_x86.h:1017
__INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1198
__INTRIN_INLINE short _InterlockedXor16(volatile short *value, short mask)
Definition: intrin_x86.h:567
__INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
Definition: intrin_x86.h:1290
__INTRIN_INLINE void _mm_mfence(void)
Definition: intrin_x86.h:104
__INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
Definition: intrin_x86.h:1521
__INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1484
GLboolean GLboolean GLboolean GLboolean a
Definition: glext.h:6204
__INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
Definition: intrin_x86.h:1469
__INTRIN_INLINE unsigned short __inword(unsigned short Port)
Definition: intrin_x86.h:1417
__INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
Definition: intrin_x86.h:1464
INT INT y
Definition: msvc.h:62
__INTRIN_INLINE void __movsw(unsigned short *Destination, const unsigned short *Source, size_t Count)
Definition: intrin_x86.h:785
__INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
Definition: intrin_x86.h:1474
__INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long *a, long b)
Definition: intrin_x86.h:711
__INTRIN_INLINE unsigned char _bittestandcomplement(long *a, long b)
Definition: intrin_x86.h:1059
__INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
Definition: intrin_x86.h:1347
INT x
Definition: msvc.h:62