ReactOS  0.4.15-dev-1392-g3014417
intrin_x86.h
Go to the documentation of this file.
1 /*
2  Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3  Microsoft Visual C++ functions. Originally developed for the ReactOS
4  (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5  projects.
6 
7  Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8 
9  Permission is hereby granted, free of charge, to any person obtaining a
10  copy of this software and associated documentation files (the "Software"),
11  to deal in the Software without restriction, including without limitation
12  the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  and/or sell copies of the Software, and to permit persons to whom the
14  Software is furnished to do so, subject to the following conditions:
15 
16  The above copyright notice and this permission notice shall be included in
17  all copies or substantial portions of the Software.
18 
19  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  DEALINGS IN THE SOFTWARE.
26 */
27 
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30 
31 /*
32  FIXME: review all "memory" clobbers, add/remove to match Visual C++
33  behavior: some "obvious" memory barriers are not present in the Visual C++
34  implementation - e.g. __stosX; on the other hand, some memory barriers that
35  *are* present could have been missed
36 */
37 
38 /*
39  NOTE: this is a *compatibility* header. Some functions may look wrong at
40  first, but they're only "as wrong" as they would be on Visual C++. Our
41  priority is compatibility
42 
43  NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44  constraints and the uses of __volatile__ out of my... hat. Do not touch
45  them. I hate cargo cult programming
46 
47  NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48  barriers aren't there in Visual C++ (e.g. __stosX)
49 
50  NOTE: review all intrinsics with a return value, add/remove __volatile__
51  where necessary. If an intrinsic whose value is ignored generates a no-op
52  under Visual C++, __volatile__ must be omitted; if it always generates code
53  (for example, if it has side effects), __volatile__ must be specified. GCC
54  will only optimize out non-volatile asm blocks with outputs, so input-only
55  blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56  and follow Visual C++ behavior
57 
58  NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59  atomic operations. Test the version like this:
60 
61  #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62  ...
63 
64  Pay attention to the type of barrier. Make it match with what Visual C++
65  would use in the same case
66 */
67 
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71 
72 /*** memcopy must be memmove ***/
73 void* __cdecl memmove(void* dest, const void* source, size_t num);
74 __INTRIN_INLINE void* __cdecl memcpy(void* dest, const void* source, size_t num)
75 {
76  return memmove(dest, source, num);
77 }
78 
79 
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
84 
85 /*** Memory barriers ***/
86 
87 #if !HAS_BUILTIN(_ReadWriteBarrier)
89 {
90  __asm__ __volatile__("" : : : "memory");
91 }
92 #endif
93 
94 /* GCC only supports full barriers */
95 #define _ReadBarrier _ReadWriteBarrier
96 #define _WriteBarrier _ReadWriteBarrier
97 
98 #if !HAS_BUILTIN(_mm_mfence)
100 {
101  __asm__ __volatile__("mfence" : : : "memory");
102 }
103 #endif
104 
105 #if !HAS_BUILTIN(_mm_lfence)
107 {
108  _ReadBarrier();
109  __asm__ __volatile__("lfence");
110  _ReadBarrier();
111 }
112 #endif
113 
114 #if !HAS_BUILTIN(_mm_sfence)
116 {
117  _WriteBarrier();
118  __asm__ __volatile__("sfence");
119  _WriteBarrier();
120 }
121 #endif
122 
123 #ifdef __x86_64__
124 __INTRIN_INLINE void __faststorefence(void)
125 {
126  long local;
127  __asm__ __volatile__("lock; orl $0, %0;" : : "m"(local));
128 }
129 #endif
130 
131 
132 /*** Atomic operations ***/
133 
134 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
135 
136 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
137 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
138 {
139  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
140 }
141 #endif
142 
143 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
144 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
145 {
146  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
147 }
148 #endif
149 
150 #if !HAS_BUILTIN(_InterlockedCompareExchange)
151 __INTRIN_INLINE long __cdecl _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
152 {
153  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
154 }
155 #endif
156 
157 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
158 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
159 {
160  return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
161 }
162 #endif
163 
164 #if !HAS_BUILTIN(_InterlockedExchange8)
165 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
166 {
167  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
168  __sync_synchronize();
169  return __sync_lock_test_and_set(Target, Value);
170 }
171 #endif
172 
173 #if !HAS_BUILTIN(_InterlockedExchange16)
174 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
175 {
176  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
177  __sync_synchronize();
178  return __sync_lock_test_and_set(Target, Value);
179 }
180 #endif
181 
182 #if !HAS_BUILTIN(_InterlockedExchange)
183 __INTRIN_INLINE long __cdecl _InterlockedExchange(volatile long * Target, long Value)
184 {
185  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
186  __sync_synchronize();
187  return __sync_lock_test_and_set(Target, Value);
188 }
189 #endif
190 
191 #if !HAS_BUILTIN(_InterlockedExchangePointer)
192 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
193 {
194  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
195  __sync_synchronize();
196  return (void *)__sync_lock_test_and_set(Target, Value);
197 }
198 #endif
199 
200 #if defined(__x86_64__)
201 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * Target, long long Value)
202 {
203  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
204  __sync_synchronize();
205  return __sync_lock_test_and_set(Target, Value);
206 }
207 #endif
208 
209 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
210 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
211 {
212  return __sync_fetch_and_add(Addend, Value);
213 }
214 #endif
215 
216 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
217 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
218 {
219  return __sync_fetch_and_add(Addend, Value);
220 }
221 #endif
222 
223 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
224 __INTRIN_INLINE long __cdecl _InterlockedExchangeAdd(volatile long * Addend, long Value)
225 {
226  return __sync_fetch_and_add(Addend, Value);
227 }
228 #endif
229 
230 #if defined(__x86_64__)
231 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * Addend, long long Value)
232 {
233  return __sync_fetch_and_add(Addend, Value);
234 }
235 #endif
236 
237 #if !HAS_BUILTIN(_InterlockedAnd8)
238 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
239 {
240  return __sync_fetch_and_and(value, mask);
241 }
242 #endif
243 
244 #if !HAS_BUILTIN(_InterlockedAnd16)
245 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
246 {
247  return __sync_fetch_and_and(value, mask);
248 }
249 #endif
250 
251 #if !HAS_BUILTIN(_InterlockedAnd)
252 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
253 {
254  return __sync_fetch_and_and(value, mask);
255 }
256 #endif
257 
258 #if defined(__x86_64__)
259 __INTRIN_INLINE long long _InterlockedAnd64(volatile long long * value, long long mask)
260 {
261  return __sync_fetch_and_and(value, mask);
262 }
263 #endif
264 
265 #if !HAS_BUILTIN(_InterlockedOr8)
266 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
267 {
268  return __sync_fetch_and_or(value, mask);
269 }
270 #endif
271 
272 #if !HAS_BUILTIN(_InterlockedOr16)
273 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
274 {
275  return __sync_fetch_and_or(value, mask);
276 }
277 #endif
278 
279 #if !HAS_BUILTIN(_InterlockedOr)
280 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
281 {
282  return __sync_fetch_and_or(value, mask);
283 }
284 #endif
285 
286 #if defined(__x86_64__)
287 __INTRIN_INLINE long long _InterlockedOr64(volatile long long * value, long long mask)
288 {
289  return __sync_fetch_and_or(value, mask);
290 }
291 #endif
292 
293 #if !HAS_BUILTIN(_InterlockedXor8)
294 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
295 {
296  return __sync_fetch_and_xor(value, mask);
297 }
298 #endif
299 
300 #if !HAS_BUILTIN(_InterlockedXor16)
301 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
302 {
303  return __sync_fetch_and_xor(value, mask);
304 }
305 #endif
306 
307 #if !HAS_BUILTIN(_InterlockedXor)
308 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
309 {
310  return __sync_fetch_and_xor(value, mask);
311 }
312 #endif
313 
314 #if defined(__x86_64__)
315 __INTRIN_INLINE long long _InterlockedXor64(volatile long long * value, long long mask)
316 {
317  return __sync_fetch_and_xor(value, mask);
318 }
319 #endif
320 
321 #if !HAS_BUILTIN(_InterlockedDecrement)
322 __INTRIN_INLINE long __cdecl _InterlockedDecrement(volatile long * lpAddend)
323 {
324  return __sync_sub_and_fetch(lpAddend, 1);
325 }
326 #endif
327 
328 #if !HAS_BUILTIN(_InterlockedIncrement)
329 __INTRIN_INLINE long __cdecl _InterlockedIncrement(volatile long * lpAddend)
330 {
331  return __sync_add_and_fetch(lpAddend, 1);
332 }
333 #endif
334 
335 #if !HAS_BUILTIN(_InterlockedDecrement16)
336 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
337 {
338  return __sync_sub_and_fetch(lpAddend, 1);
339 }
340 #endif
341 
342 #if !HAS_BUILTIN(_InterlockedIncrement16)
343 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
344 {
345  return __sync_add_and_fetch(lpAddend, 1);
346 }
347 #endif
348 
349 #if defined(__x86_64__)
350 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
351 {
352  return __sync_sub_and_fetch(lpAddend, 1);
353 }
354 
355 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
356 {
357  return __sync_add_and_fetch(lpAddend, 1);
358 }
359 #endif
360 
361 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
362 
363 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
364 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
365 {
366  char retval = Comperand;
367  __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
368  return retval;
369 }
370 #endif
371 
372 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
373 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
374 {
375  short retval = Comperand;
376  __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
377  return retval;
378 }
379 #endif
380 
381 #if !HAS_BUILTIN(_InterlockedCompareExchange)
382 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
383 {
384  long retval = Comperand;
385  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
386  return retval;
387 }
388 #endif
389 
390 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
391 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
392 {
393  void * retval = (void *)Comperand;
394  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
395  return retval;
396 }
397 #endif
398 
399 #if !HAS_BUILTIN(_InterlockedExchange8)
401 {
402  char retval = Value;
403  __asm__("xchgb %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
404  return retval;
405 }
406 #endif
407 
408 #if !HAS_BUILTIN(_InterlockedExchange16)
409 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
410 {
411  short retval = Value;
412  __asm__("xchgw %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
413  return retval;
414 }
415 #endif
416 
417 #if !HAS_BUILTIN(_InterlockedExchange)
419 {
420  long retval = Value;
421  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
422  return retval;
423 }
424 #endif
425 
426 #if !HAS_BUILTIN(_InterlockedExchangePointer)
427 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
428 {
429  void * retval = Value;
430  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
431  return retval;
432 }
433 #endif
434 
435 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
437 {
438  char retval = Value;
439  __asm__("lock; xaddb %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
440  return retval;
441 }
442 #endif
443 
444 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
446 {
447  short retval = Value;
448  __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
449  return retval;
450 }
451 #endif
452 
453 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
455 {
456  long retval = Value;
457  __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
458  return retval;
459 }
460 #endif
461 
462 #if !HAS_BUILTIN(_InterlockedAnd8)
463 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
464 {
465  char x;
466  char y;
467 
468  y = *value;
469 
470  do
471  {
472  x = y;
474  }
475  while(y != x);
476 
477  return y;
478 }
479 #endif
480 
481 #if !HAS_BUILTIN(_InterlockedAnd16)
482 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
483 {
484  short x;
485  short y;
486 
487  y = *value;
488 
489  do
490  {
491  x = y;
493  }
494  while(y != x);
495 
496  return y;
497 }
498 #endif
499 
500 #if !HAS_BUILTIN(_InterlockedAnd)
501 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
502 {
503  long x;
504  long y;
505 
506  y = *value;
507 
508  do
509  {
510  x = y;
512  }
513  while(y != x);
514 
515  return y;
516 }
517 #endif
518 
519 #if !HAS_BUILTIN(_InterlockedOr8)
520 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
521 {
522  char x;
523  char y;
524 
525  y = *value;
526 
527  do
528  {
529  x = y;
531  }
532  while(y != x);
533 
534  return y;
535 }
536 #endif
537 
538 #if !HAS_BUILTIN(_InterlockedOr16)
539 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
540 {
541  short x;
542  short y;
543 
544  y = *value;
545 
546  do
547  {
548  x = y;
550  }
551  while(y != x);
552 
553  return y;
554 }
555 #endif
556 
557 #if !HAS_BUILTIN(_InterlockedOr)
558 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
559 {
560  long x;
561  long y;
562 
563  y = *value;
564 
565  do
566  {
567  x = y;
569  }
570  while(y != x);
571 
572  return y;
573 }
574 #endif
575 
576 #if !HAS_BUILTIN(_InterlockedXor8)
577 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
578 {
579  char x;
580  char y;
581 
582  y = *value;
583 
584  do
585  {
586  x = y;
588  }
589  while(y != x);
590 
591  return y;
592 }
593 #endif
594 
595 #if !HAS_BUILTIN(_InterlockedXor16)
596 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
597 {
598  short x;
599  short y;
600 
601  y = *value;
602 
603  do
604  {
605  x = y;
607  }
608  while(y != x);
609 
610  return y;
611 }
612 #endif
613 
614 #if !HAS_BUILTIN(_InterlockedXor)
615 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
616 {
617  long x;
618  long y;
619 
620  y = *value;
621 
622  do
623  {
624  x = y;
626  }
627  while(y != x);
628 
629  return y;
630 }
631 #endif
632 
633 #if !HAS_BUILTIN(_InterlockedDecrement)
634 __INTRIN_INLINE long _InterlockedDecrement(volatile long * lpAddend)
635 {
636  return _InterlockedExchangeAdd(lpAddend, -1) - 1;
637 }
638 #endif
639 
640 #if !HAS_BUILTIN(_InterlockedIncrement)
641 __INTRIN_INLINE long _InterlockedIncrement(volatile long * lpAddend)
642 {
643  return _InterlockedExchangeAdd(lpAddend, 1) + 1;
644 }
645 #endif
646 
647 #if !HAS_BUILTIN(_InterlockedDecrement16)
648 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
649 {
650  return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
651 }
652 #endif
653 
654 #if !HAS_BUILTIN(_InterlockedIncrement16)
655 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
656 {
657  return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
658 }
659 #endif
660 
661 #if defined(__x86_64__)
662 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
663 {
664  return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
665 }
666 
667 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
668 {
669  return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
670 }
671 #endif
672 
673 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
674 
675 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
676 
677 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
678 {
679  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
680 }
681 
682 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
683 
684 #if !HAS_BUILTIN(_InterlockedCompareExchange64)
685 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
686 {
687  long long retval = Comperand;
688 
689  __asm__
690  (
691  "lock; cmpxchg8b %[Destination]" :
692  [retval] "+A" (retval) :
693  [Destination] "m" (*Destination),
694  "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
695  "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
696  "memory"
697  );
698 
699  return retval;
700 }
701 #endif
702 
703 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
704 
705 #ifdef __i386__
706 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * Addend, long Value)
707 {
708  __asm__
709  (
710  "lock; addl %[Value], %[Lo32];"
711  "jae LABEL%=;"
712  "lock; adcl $0, %[Hi32];"
713  "LABEL%=:;" :
714  [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
715  [Value] "ir" (Value) :
716  "memory"
717  );
718 
719  return Value;
720 }
721 #endif /* __i386__ */
722 
723 #if !HAS_BUILTIN(_interlockedbittestandreset)
724 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, long b)
725 {
726  unsigned char retval;
727  __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
728  return retval;
729 }
730 #endif
731 
732 #if defined(__x86_64__)
733 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, long long b)
734 {
735  unsigned char retval;
736  __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
737  return retval;
738 }
739 #endif
740 
741 #if !HAS_BUILTIN(_interlockedbittestandset)
742 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, long b)
743 {
744  unsigned char retval;
745  __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
746  return retval;
747 }
748 #endif
749 
750 #if defined(__x86_64__)
751 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, long long b)
752 {
753  unsigned char retval;
754  __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
755  return retval;
756 }
757 #endif
758 
759 /*** String operations ***/
760 
761 #if !HAS_BUILTIN(__stosb)
762 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
763 __INTRIN_INLINE void __stosb(unsigned char * Dest, unsigned char Data, size_t Count)
764 {
765  __asm__ __volatile__
766  (
767  "rep; stosb" :
768  [Dest] "=D" (Dest), [Count] "=c" (Count) :
769  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
770  );
771 }
772 #endif
773 
774 __INTRIN_INLINE void __stosw(unsigned short * Dest, unsigned short Data, size_t Count)
775 {
776  __asm__ __volatile__
777  (
778  "rep; stosw" :
779  [Dest] "=D" (Dest), [Count] "=c" (Count) :
780  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
781  );
782 }
783 
784 __INTRIN_INLINE void __stosd(unsigned long * Dest, unsigned long Data, size_t Count)
785 {
786  __asm__ __volatile__
787  (
788  "rep; stosl" :
789  [Dest] "=D" (Dest), [Count] "=c" (Count) :
790  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
791  );
792 }
793 
794 #ifdef __x86_64__
795 __INTRIN_INLINE void __stosq(unsigned long long * Dest, unsigned long long Data, size_t Count)
796 {
797  __asm__ __volatile__
798  (
799  "rep; stosq" :
800  [Dest] "=D" (Dest), [Count] "=c" (Count) :
801  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
802  );
803 }
804 #endif
805 
806 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
807 {
808  __asm__ __volatile__
809  (
810  "rep; movsb" :
811  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
812  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
813  );
814 }
815 
816 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
817 {
818  __asm__ __volatile__
819  (
820  "rep; movsw" :
821  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
822  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
823  );
824 }
825 
826 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
827 {
828  __asm__ __volatile__
829  (
830  "rep; movsd" :
831  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
832  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
833  );
834 }
835 
836 #ifdef __x86_64__
837 __INTRIN_INLINE void __movsq(unsigned long * Destination, const unsigned long * Source, size_t Count)
838 {
839  __asm__ __volatile__
840  (
841  "rep; movsq" :
842  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
843  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
844  );
845 }
846 #endif
847 
848 #if defined(__x86_64__)
849 
850 /*** GS segment addressing ***/
851 
852 __INTRIN_INLINE void __writegsbyte(unsigned long Offset, unsigned char Data)
853 {
854  __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
855 }
856 
857 __INTRIN_INLINE void __writegsword(unsigned long Offset, unsigned short Data)
858 {
859  __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
860 }
861 
862 __INTRIN_INLINE void __writegsdword(unsigned long Offset, unsigned long Data)
863 {
864  __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
865 }
866 
867 __INTRIN_INLINE void __writegsqword(unsigned long Offset, unsigned long long Data)
868 {
869  __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
870 }
871 
872 __INTRIN_INLINE unsigned char __readgsbyte(unsigned long Offset)
873 {
874  unsigned char value;
875  __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
876  return value;
877 }
878 
879 __INTRIN_INLINE unsigned short __readgsword(unsigned long Offset)
880 {
881  unsigned short value;
882  __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
883  return value;
884 }
885 
886 __INTRIN_INLINE unsigned long __readgsdword(unsigned long Offset)
887 {
888  unsigned long value;
889  __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
890  return value;
891 }
892 
893 __INTRIN_INLINE unsigned long long __readgsqword(unsigned long Offset)
894 {
895  unsigned long long value;
896  __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
897  return value;
898 }
899 
900 __INTRIN_INLINE void __incgsbyte(unsigned long Offset)
901 {
902  __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
903 }
904 
905 __INTRIN_INLINE void __incgsword(unsigned long Offset)
906 {
907  __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
908 }
909 
910 __INTRIN_INLINE void __incgsdword(unsigned long Offset)
911 {
912  __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
913 }
914 
915 __INTRIN_INLINE void __incgsqword(unsigned long Offset)
916 {
917  __asm__ __volatile__("incq %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
918 }
919 
920 __INTRIN_INLINE void __addgsbyte(unsigned long Offset, unsigned char Data)
921 {
922  __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
923 }
924 
925 __INTRIN_INLINE void __addgsword(unsigned long Offset, unsigned short Data)
926 {
927  __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
928 }
929 
930 __INTRIN_INLINE void __addgsdword(unsigned long Offset, unsigned int Data)
931 {
932  __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
933 }
934 
935 __INTRIN_INLINE void __addgsqword(unsigned long Offset, unsigned long long Data)
936 {
937  __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
938 }
939 
940 #else /* defined(__x86_64__) */
941 
942 /*** FS segment addressing ***/
943 
944 __INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
945 {
946  __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
947 }
948 
949 __INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
950 {
951  __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
952 }
953 
954 __INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
955 {
956  __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
957 }
958 
959 #if !HAS_BUILTIN(__readfsbyte)
960 __INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
961 {
962  unsigned char value;
963  __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
964  return value;
965 }
966 #endif
967 
968 #if !HAS_BUILTIN(__readfsword)
969 __INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
970 {
971  unsigned short value;
972  __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
973  return value;
974 }
975 #endif
976 
977 #if !HAS_BUILTIN(__readfsdword)
978 __INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
979 {
980  unsigned long value;
981  __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
982  return value;
983 }
984 #endif
985 
986 __INTRIN_INLINE void __incfsbyte(unsigned long Offset)
987 {
988  __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
989 }
990 
991 __INTRIN_INLINE void __incfsword(unsigned long Offset)
992 {
993  __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
994 }
995 
997 {
998  __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
999 }
1000 
1001 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
1002 __INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
1003 {
1004  if(!__builtin_constant_p(Offset))
1005  __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1006  else
1007  __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1008 }
1009 
1010 __INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
1011 {
1012  if(!__builtin_constant_p(Offset))
1013  __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1014  else
1015  __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1016 }
1017 
1018 __INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
1019 {
1020  if(!__builtin_constant_p(Offset))
1021  __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1022  else
1023  __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1024 }
1025 
1026 #endif /* defined(__x86_64__) */
1027 
1028 
1029 /*** Bit manipulation ***/
1030 
1031 #if !HAS_BUILTIN(_BitScanForward)
1032 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * Index, unsigned long Mask)
1033 {
1034  __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1035  return Mask ? 1 : 0;
1036 }
1037 #endif
1038 
1039 #if !HAS_BUILTIN(_BitScanReverse)
1040 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * Index, unsigned long Mask)
1041 {
1042  __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1043  return Mask ? 1 : 0;
1044 }
1045 #endif
1046 
1047 #if !HAS_BUILTIN(_bittest)
1048 /* NOTE: again, the bizarre implementation follows Visual C++ */
1049 __INTRIN_INLINE unsigned char _bittest(const long * a, long b)
1050 {
1051  unsigned char retval;
1052 
1053  if(__builtin_constant_p(b))
1054  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
1055  else
1056  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1057 
1058  return retval;
1059 }
1060 #endif
1061 
1062 #ifdef __x86_64__
1063 __INTRIN_INLINE unsigned char _BitScanForward64(unsigned long * Index, unsigned long long Mask)
1064 {
1065  unsigned long long Index64;
1066  __asm__("bsfq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1067  *Index = Index64;
1068  return Mask ? 1 : 0;
1069 }
1070 
1071 __INTRIN_INLINE unsigned char _BitScanReverse64(unsigned long * Index, unsigned long long Mask)
1072 {
1073  unsigned long long Index64;
1074  __asm__("bsrq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1075  *Index = Index64;
1076  return Mask ? 1 : 0;
1077 }
1078 
1079 __INTRIN_INLINE unsigned char _bittest64(const long long * a, long long b)
1080 {
1081  unsigned char retval;
1082 
1083  if(__builtin_constant_p(b))
1084  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
1085  else
1086  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1087 
1088  return retval;
1089 }
1090 #endif
1091 
1092 #if !HAS_BUILTIN(_bittestandcomplement)
1093 __INTRIN_INLINE unsigned char _bittestandcomplement(long * a, long b)
1094 {
1095  unsigned char retval;
1096 
1097  if(__builtin_constant_p(b))
1098  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1099  else
1100  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1101 
1102  return retval;
1103 }
1104 #endif
1105 
1106 #if !HAS_BUILTIN(_bittestandreset)
1107 __INTRIN_INLINE unsigned char _bittestandreset(long * a, long b)
1108 {
1109  unsigned char retval;
1110 
1111  if(__builtin_constant_p(b))
1112  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1113  else
1114  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1115 
1116  return retval;
1117 }
1118 #endif
1119 
1120 #if !HAS_BUILTIN(_bittestandset)
1121 __INTRIN_INLINE unsigned char _bittestandset(long * a, long b)
1122 {
1123  unsigned char retval;
1124 
1125  if(__builtin_constant_p(b))
1126  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1127  else
1128  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1129 
1130  return retval;
1131 }
1132 #endif
1133 
1134 #ifdef __x86_64__
1135 
1136 __INTRIN_INLINE unsigned char _bittestandset64(long long * a, long long b)
1137 {
1138  unsigned char retval;
1139 
1140  if(__builtin_constant_p(b))
1141  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1142  else
1143  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1144 
1145  return retval;
1146 }
1147 
1148 __INTRIN_INLINE unsigned char _bittestandreset64(long long * a, long long b)
1149 {
1150  unsigned char retval;
1151 
1152  if(__builtin_constant_p(b))
1153  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1154  else
1155  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1156 
1157  return retval;
1158 }
1159 
1160 __INTRIN_INLINE unsigned char _bittestandcomplement64(long long * a, long long b)
1161 {
1162  unsigned char retval;
1163 
1164  if(__builtin_constant_p(b))
1165  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1166  else
1167  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1168 
1169  return retval;
1170 }
1171 
1172 #endif /* __x86_64__ */
1173 
1174 #if !HAS_BUILTIN(_rotl8)
1175 __INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
1176 {
1177  unsigned char retval;
1178  __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1179  return retval;
1180 }
1181 #endif
1182 
1183 #if !HAS_BUILTIN(_rotl16)
1184 __INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
1185 {
1186  unsigned short retval;
1187  __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1188  return retval;
1189 }
1190 #endif
1191 
1192 #if !HAS_BUILTIN(_rotl)
1193 __INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
1194 {
1195  unsigned int retval;
1196  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1197  return retval;
1198 }
1199 #endif
1200 
1201 #ifdef __x86_64__
1202 __INTRIN_INLINE unsigned long long _rotl64(unsigned long long value, int shift)
1203 {
1204  unsigned long long retval;
1205  __asm__("rolq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1206  return retval;
1207 }
1208 #else /* __x86_64__ */
1209 #if !HAS_BUILTIN(_rotl64)
1210 __INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
1211 {
1212  /* FIXME: this is probably not optimal */
1213  return (value << shift) | (value >> (64 - shift));
1214 }
1215 #endif /* !HAS_BUILTIN(_rotl64) */
1216 #endif /* __x86_64__ */
1217 
1218 #if !HAS_BUILTIN(_rotr)
1219 __INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
1220 {
1221  unsigned int retval;
1222  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1223  return retval;
1224 }
1225 #endif
1226 
1227 #if !HAS_BUILTIN(_rotr8)
1228 __INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
1229 {
1230  unsigned char retval;
1231  __asm__("rorb %b[shift], %b[retval]" : [retval] "=qm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1232  return retval;
1233 }
1234 #endif
1235 
1236 #if !HAS_BUILTIN(_rotr16)
1237 __INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
1238 {
1239  unsigned short retval;
1240  __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1241  return retval;
1242 }
1243 #endif
1244 
1245 #ifdef __x86_64__
1246 __INTRIN_INLINE unsigned long long _rotr64(unsigned long long value, int shift)
1247 {
1248  unsigned long long retval;
1249  __asm__("rorq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1250  return retval;
1251 }
1252 #else /* __x86_64__ */
1253 #if !HAS_BUILTIN(_rotr64)
1254 __INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
1255 {
1256  /* FIXME: this is probably not optimal */
1257  return (value >> shift) | (value << (64 - shift));
1258 }
1259 #endif /* !HAS_BUILTIN(_rotr64) */
1260 #endif /* __x86_64__ */
1261 
1262 #if !HAS_BUILTIN(_lrotl)
1263 __INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
1264 {
1265  unsigned long retval;
1266  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1267  return retval;
1268 }
1269 #endif
1270 
1271 #if !HAS_BUILTIN(_lrotr)
1272 __INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
1273 {
1274  unsigned long retval;
1275  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1276  return retval;
1277 }
1278 #endif
1279 
1280 /*
1281  NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1282  constraint (edx:eax) for the Mask argument, because it's the only way GCC
1283  can pass 64-bit operands around - passing the two 32 bit parts separately
1284  just confuses it. Also we declare Bit as an int and then truncate it to
1285  match Visual C++ behavior
1286 */
1287 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1288 {
1289  unsigned long long retval = Mask;
1290 
1291  __asm__
1292  (
1293  "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1294  "+A" (retval) :
1295  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1296  );
1297 
1298  return retval;
1299 }
1300 
1301 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1302 {
1303  long long retval = Mask;
1304 
1305  __asm__
1306  (
1307  "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1308  "+A" (retval) :
1309  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1310  );
1311 
1312  return retval;
1313 }
1314 
1315 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1316 {
1317  unsigned long long retval = Mask;
1318 
1319  __asm__
1320  (
1321  "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1322  "+A" (retval) :
1323  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1324  );
1325 
1326  return retval;
1327 }
1328 
1329 __INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
1330 {
1331  unsigned short retval;
1332  __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
1333  return retval;
1334 }
1335 
1336 __INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
1337 {
1338  unsigned long retval;
1339  __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1340  return retval;
1341 }
1342 
1343 #ifdef __x86_64__
1344 __INTRIN_INLINE unsigned long long _byteswap_uint64(unsigned long long value)
1345 {
1346  unsigned long long retval;
1347  __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1348  return retval;
1349 }
1350 #else
1351 __INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
1352 {
1353  union {
1354  unsigned long long int64part;
1355  struct {
1356  unsigned long lowpart;
1357  unsigned long hipart;
1358  };
1359  } retval;
1360  retval.int64part = value;
1361  __asm__("bswapl %[lowpart]\n"
1362  "bswapl %[hipart]\n"
1363  : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
1364  return retval.int64part;
1365 }
1366 #endif
1367 
1368 #if !HAS_BUILTIN(__lzcnt)
1369 __INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
1370 {
1371  return __builtin_clz(value);
1372 }
1373 #endif
1374 
1375 #if !HAS_BUILTIN(__lzcnt16)
1376 __INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
1377 {
1378  return __builtin_clz(value);
1379 }
1380 #endif
1381 
1382 #if !HAS_BUILTIN(__popcnt)
1383 __INTRIN_INLINE unsigned int __popcnt(unsigned int value)
1384 {
1385  return __builtin_popcount(value);
1386 }
1387 #endif
1388 
1389 #if !HAS_BUILTIN(__popcnt16)
1390 __INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
1391 {
1392  return __builtin_popcount(value);
1393 }
1394 #endif
1395 
1396 #ifdef __x86_64__
1397 __INTRIN_INLINE unsigned long long __lzcnt64(unsigned long long value)
1398 {
1399  return __builtin_clzll(value);
1400 }
1401 
1402 __INTRIN_INLINE unsigned long long __popcnt64(unsigned long long value)
1403 {
1404  return __builtin_popcountll(value);
1405 }
1406 #endif
1407 
1408 /*** 64-bit math ***/
1409 
1410 #if !HAS_BUILTIN(__emul)
1411 __INTRIN_INLINE long long __emul(int a, int b)
1412 {
1413  long long retval;
1414  __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1415  return retval;
1416 }
1417 #endif
1418 
1419 #if !HAS_BUILTIN(__emulu)
1420 __INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
1421 {
1422  unsigned long long retval;
1423  __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1424  return retval;
1425 }
1426 #endif
1427 
1428 __INTRIN_INLINE long long __cdecl _abs64(long long value)
1429 {
1430  return (value >= 0) ? value : -value;
1431 }
1432 
1433 #ifdef __x86_64__
1434 
1435 __INTRIN_INLINE long long __mulh(long long a, long long b)
1436 {
1437  long long retval;
1438  __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1439  return retval;
1440 }
1441 
1442 __INTRIN_INLINE unsigned long long __umulh(unsigned long long a, unsigned long long b)
1443 {
1444  unsigned long long retval;
1445  __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1446  return retval;
1447 }
1448 
1449 #endif
1450 
1451 /*** Port I/O ***/
1452 
1453 __INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
1454 {
1455  unsigned char byte;
1456  __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1457  return byte;
1458 }
1459 
1460 __INTRIN_INLINE unsigned short __inword(unsigned short Port)
1461 {
1462  unsigned short word;
1463  __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1464  return word;
1465 }
1466 
1467 __INTRIN_INLINE unsigned long __indword(unsigned short Port)
1468 {
1469  unsigned long dword;
1470  __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1471  return dword;
1472 }
1473 
1474 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1475 {
1476  __asm__ __volatile__
1477  (
1478  "rep; insb" :
1479  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1480  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1481  "memory"
1482  );
1483 }
1484 
1485 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1486 {
1487  __asm__ __volatile__
1488  (
1489  "rep; insw" :
1490  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1491  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1492  "memory"
1493  );
1494 }
1495 
1496 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1497 {
1498  __asm__ __volatile__
1499  (
1500  "rep; insl" :
1501  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1502  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1503  "memory"
1504  );
1505 }
1506 
1507 __INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
1508 {
1509  __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1510 }
1511 
1512 __INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
1513 {
1514  __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1515 }
1516 
1517 __INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
1518 {
1519  __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1520 }
1521 
1522 __INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1523 {
1524  __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1525 }
1526 
1527 __INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1528 {
1529  __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1530 }
1531 
1532 __INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1533 {
1534  __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1535 }
1536 
1537 __INTRIN_INLINE int __cdecl _inp(unsigned short Port)
1538 {
1539  return __inbyte(Port);
1540 }
1541 
1542 __INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
1543 {
1544  return __inword(Port);
1545 }
1546 
1547 __INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
1548 {
1549  return __indword(Port);
1550 }
1551 
1552 __INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
1553 {
1554  __outbyte(Port, (unsigned char)databyte);
1555  return databyte;
1556 }
1557 
1558 __INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
1559 {
1560  __outword(Port, dataword);
1561  return dataword;
1562 }
1563 
1564 __INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
1565 {
1566  __outdword(Port, dataword);
1567  return dataword;
1568 }
1569 
1570 
1571 /*** System information ***/
1572 
1573 __INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
1574 {
1575  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1576 }
1577 
1578 __INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
1579 {
1580  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType), "c" (ECXValue));
1581 }
1582 
1583 #if !HAS_BUILTIN(__rdtsc)
1584 __INTRIN_INLINE unsigned long long __rdtsc(void)
1585 {
1586 #ifdef __x86_64__
1587  unsigned long long low, high;
1588  __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1589  return low | (high << 32);
1590 #else
1591  unsigned long long retval;
1592  __asm__ __volatile__("rdtsc" : "=A"(retval));
1593  return retval;
1594 #endif
1595 }
1596 #endif /* !HAS_BUILTIN(__rdtsc) */
1597 
1599 {
1600  __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1601 }
1602 
1604 {
1605  uintptr_t retval;
1606  __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1607  return retval;
1608 }
1609 
1610 /*** Interrupts ***/
1611 
1612 #if !HAS_BUILTIN(__debugbreak)
1614 {
1615  __asm__("int $3");
1616 }
1617 #endif
1618 
1619 #if !HAS_BUILTIN(__ud2)
1621 {
1622  __asm__("ud2");
1623 }
1624 #endif
1625 
1626 #if !HAS_BUILTIN(__int2c)
1628 {
1629  __asm__("int $0x2c");
1630 }
1631 #endif
1632 
1634 {
1635  __asm__("cli" : : : "memory");
1636 }
1637 
1639 {
1640  __asm__("sti" : : : "memory");
1641 }
1642 
1644 {
1645  __asm__("hlt" : : : "memory");
1646 }
1647 
1648 #if !HAS_BUILTIN(__fastfail)
1649 __declspec(noreturn)
1650 __INTRIN_INLINE void __fastfail(unsigned int Code)
1651 {
1652  __asm__("int $0x29" : : "c"(Code) : "memory");
1653  __builtin_unreachable();
1654 }
1655 #endif
1656 
1657 /*** Protected memory management ***/
1658 
1659 #ifdef __x86_64__
1660 
1661 __INTRIN_INLINE void __writecr0(unsigned long long Data)
1662 {
1663  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1664 }
1665 
1666 __INTRIN_INLINE void __writecr3(unsigned long long Data)
1667 {
1668  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1669 }
1670 
1671 __INTRIN_INLINE void __writecr4(unsigned long long Data)
1672 {
1673  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1674 }
1675 
1676 __INTRIN_INLINE void __writecr8(unsigned long long Data)
1677 {
1678  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1679 }
1680 
1681 __INTRIN_INLINE unsigned long long __readcr0(void)
1682 {
1683  unsigned long long value;
1684  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1685  return value;
1686 }
1687 
1688 __INTRIN_INLINE unsigned long long __readcr2(void)
1689 {
1690  unsigned long long value;
1691  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1692  return value;
1693 }
1694 
1695 __INTRIN_INLINE unsigned long long __readcr3(void)
1696 {
1697  unsigned long long value;
1698  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1699  return value;
1700 }
1701 
1702 __INTRIN_INLINE unsigned long long __readcr4(void)
1703 {
1704  unsigned long long value;
1705  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1706  return value;
1707 }
1708 
1709 __INTRIN_INLINE unsigned long long __readcr8(void)
1710 {
1711  unsigned long long value;
1712  __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1713  return value;
1714 }
1715 
1716 #else /* __x86_64__ */
1717 
1718 __INTRIN_INLINE void __writecr0(unsigned int Data)
1719 {
1720  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1721 }
1722 
1723 __INTRIN_INLINE void __writecr3(unsigned int Data)
1724 {
1725  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1726 }
1727 
1728 __INTRIN_INLINE void __writecr4(unsigned int Data)
1729 {
1730  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1731 }
1732 
1733 __INTRIN_INLINE unsigned long __readcr0(void)
1734 {
1735  unsigned long value;
1736  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1737  return value;
1738 }
1739 
1740 __INTRIN_INLINE unsigned long __readcr2(void)
1741 {
1742  unsigned long value;
1743  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1744  return value;
1745 }
1746 
1747 __INTRIN_INLINE unsigned long __readcr3(void)
1748 {
1749  unsigned long value;
1750  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1751  return value;
1752 }
1753 
1754 __INTRIN_INLINE unsigned long __readcr4(void)
1755 {
1756  unsigned long value;
1757  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1758  return value;
1759 }
1760 
1761 #endif /* __x86_64__ */
1762 
1763 #ifdef __x86_64__
1764 
1765 __INTRIN_INLINE unsigned long long __readdr(unsigned int reg)
1766 {
1767  unsigned long long value;
1768  switch (reg)
1769  {
1770  case 0:
1771  __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1772  break;
1773  case 1:
1774  __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1775  break;
1776  case 2:
1777  __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1778  break;
1779  case 3:
1780  __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1781  break;
1782  case 4:
1783  __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1784  break;
1785  case 5:
1786  __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1787  break;
1788  case 6:
1789  __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1790  break;
1791  case 7:
1792  __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1793  break;
1794  }
1795  return value;
1796 }
1797 
1798 __INTRIN_INLINE void __writedr(unsigned reg, unsigned long long value)
1799 {
1800  switch (reg)
1801  {
1802  case 0:
1803  __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1804  break;
1805  case 1:
1806  __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1807  break;
1808  case 2:
1809  __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1810  break;
1811  case 3:
1812  __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1813  break;
1814  case 4:
1815  __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1816  break;
1817  case 5:
1818  __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1819  break;
1820  case 6:
1821  __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1822  break;
1823  case 7:
1824  __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1825  break;
1826  }
1827 }
1828 
1829 #else /* __x86_64__ */
1830 
1831 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1832 {
1833  unsigned int value;
1834  switch (reg)
1835  {
1836  case 0:
1837  __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1838  break;
1839  case 1:
1840  __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1841  break;
1842  case 2:
1843  __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1844  break;
1845  case 3:
1846  __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1847  break;
1848  case 4:
1849  __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1850  break;
1851  case 5:
1852  __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1853  break;
1854  case 6:
1855  __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1856  break;
1857  case 7:
1858  __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1859  break;
1860  }
1861  return value;
1862 }
1863 
1864 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1865 {
1866  switch (reg)
1867  {
1868  case 0:
1869  __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1870  break;
1871  case 1:
1872  __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1873  break;
1874  case 2:
1875  __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1876  break;
1877  case 3:
1878  __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1879  break;
1880  case 4:
1881  __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1882  break;
1883  case 5:
1884  __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1885  break;
1886  case 6:
1887  __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1888  break;
1889  case 7:
1890  __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1891  break;
1892  }
1893 }
1894 
1895 #endif /* __x86_64__ */
1896 
1898 {
1899  __asm__ __volatile__ ("invlpg (%[Address])" : : [Address] "b" (Address) : "memory");
1900 }
1901 
1902 
1903 /*** System operations ***/
1904 
1905 __INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
1906 {
1907 #ifdef __x86_64__
1908  unsigned long low, high;
1909  __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1910  return ((unsigned long long)high << 32) | low;
1911 #else
1912  unsigned long long retval;
1913  __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1914  return retval;
1915 #endif
1916 }
1917 
1918 __INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
1919 {
1920 #ifdef __x86_64__
1921  __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1922 #else
1923  __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1924 #endif
1925 }
1926 
1927 __INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
1928 {
1929  unsigned long long retval;
1930  __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
1931  return retval;
1932 }
1933 
1934 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1935 __INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
1936 {
1937  unsigned long retval;
1938  __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
1939  return retval;
1940 }
1941 
1943 {
1944  __asm__ __volatile__("wbinvd" : : : "memory");
1945 }
1946 
1948 {
1949  __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
1950 }
1951 
1953 {
1954  __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
1955 }
1956 
1958 {
1959  __asm__ __volatile__("sgdt %0" : : "m"(*(short*)Destination) : "memory");
1960 }
1961 
1962 /*** Misc operations ***/
1963 
1964 #if !HAS_BUILTIN(_mm_pause)
1966 {
1967  __asm__ __volatile__("pause" : : : "memory");
1968 }
1969 #endif
1970 
1972 {
1973  __asm__ __volatile__("nop");
1974 }
1975 
1976 #ifdef __cplusplus
1977 }
1978 #endif
1979 
1980 #endif /* KJK_INTRIN_X86_H_ */
1981 
1982 /* EOF */
__INTRIN_INLINE void __ud2(void)
Definition: intrin_x86.h:1620
CPPORT Port[4]
Definition: headless.c:34
__INTRIN_INLINE void __writecr4(unsigned int Data)
Definition: intrin_x86.h:1728
__INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
Definition: intrin_x86.h:1542
__INTRIN_INLINE void __stosd(unsigned long *Dest, unsigned long Data, size_t Count)
Definition: intrin_x86.h:784
__INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
Definition: intrin_x86.h:1864
__INTRIN_INLINE void _mm_lfence(void)
Definition: intrin_x86.h:106
__INTRIN_INLINE void __stosb(unsigned char *Dest, unsigned char Data, size_t Count)
Definition: intrin_x86.h:763
__INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
Definition: intrin_x86.h:1210
#define shift
Definition: input.c:1756
__INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:1002
__INTRIN_INLINE void *__cdecl memcpy(void *dest, const void *source, size_t num)
Definition: intrin_x86.h:74
__INTRIN_INLINE char _InterlockedCompareExchange8(volatile char *Destination, char Exchange, char Comperand)
Definition: intrin_x86.h:364
__INTRIN_INLINE unsigned long __readcr2(void)
Definition: intrin_x86.h:1740
__INTRIN_INLINE void __movsb(unsigned char *Destination, const unsigned char *Source, size_t Count)
Definition: intrin_x86.h:806
__INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
Definition: intrin_x86.h:1905
__INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long *a, long b)
Definition: intrin_x86.h:724
#define __cdecl
Definition: accygwin.h:79
__INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:944
__INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1184
__INTRIN_INLINE void __incfsdword(unsigned long Offset)
Definition: intrin_x86.h:996
__INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
Definition: intrin_x86.h:978
__INTRIN_INLINE unsigned char _BitScanReverse(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1040
__INTRIN_INLINE void __wbinvd(void)
Definition: intrin_x86.h:1942
__INTRIN_INLINE long _InterlockedXor(volatile long *value, long mask)
Definition: intrin_x86.h:615
__INTRIN_INLINE void __incfsword(unsigned long Offset)
Definition: intrin_x86.h:991
__INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
Definition: intrin_x86.h:1272
__INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1474
__INTRIN_INLINE void __halt(void)
Definition: intrin_x86.h:1643
__INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long *Destination, long long Exchange, long long Comperand)
Definition: intrin_x86.h:685
__INTRIN_INLINE void __stosw(unsigned short *Dest, unsigned short Data, size_t Count)
Definition: intrin_x86.h:774
__INTRIN_INLINE void __lidt(void *Source)
Definition: intrin_x86.h:1947
__INTRIN_INLINE unsigned int __popcnt(unsigned int value)
Definition: intrin_x86.h:1383
GLint GLint GLint GLint GLint x
Definition: gl.h:1548
__INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1315
__INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
Definition: intrin_x86.h:1935
__declspec(noreturn) __INTRIN_INLINE void __fastfail(unsigned int Code)
Definition: intrin_x86.h:1649
__INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
Definition: intrin_x86.h:1351
__INTRIN_INLINE unsigned long __readcr3(void)
Definition: intrin_x86.h:1747
__INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
Definition: intrin_x86.h:960
__INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1496
__INTRIN_INLINE void __movsd(unsigned long *Destination, const unsigned long *Source, size_t Count)
Definition: intrin_x86.h:826
__INTRIN_INLINE long long __emul(int a, int b)
Definition: intrin_x86.h:1411
IN OUT PLONG Addend
Definition: CrNtStubs.h:22
__INTRIN_INLINE short _InterlockedIncrement16(volatile short *lpAddend)
Definition: intrin_x86.h:655
__INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:949
__INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
Definition: intrin_x86.h:1193
__INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
Definition: intrin_x86.h:1254
__INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
Definition: intrin_x86.h:1369
__INTRIN_INLINE unsigned char _BitScanForward(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1032
__INTRIN_INLINE unsigned char _bittestandset(long *a, long b)
Definition: intrin_x86.h:1121
__INTRIN_INLINE long _InterlockedOr(volatile long *value, long mask)
Definition: intrin_x86.h:558
__INTRIN_INLINE int __cdecl _inp(unsigned short Port)
Definition: intrin_x86.h:1537
__INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
Definition: intrin_x86.h:1336
__INTRIN_INLINE void __cdecl __debugbreak(void)
Definition: intrin_x86.h:1613
__INTRIN_INLINE void __writecr3(unsigned int Data)
Definition: intrin_x86.h:1723
__INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1522
__INTRIN_INLINE unsigned char _bittestandreset(long *a, long b)
Definition: intrin_x86.h:1107
GLenum GLint GLuint mask
Definition: glext.h:6028
__INTRIN_INLINE uintptr_t __readeflags(void)
Definition: intrin_x86.h:1603
__INTRIN_INLINE long _InterlockedIncrement(volatile long *lpAddend)
Definition: intrin_x86.h:641
__asm__("\t.globl GetPhys\n" "GetPhys:\t\n" "mflr 0\n\t" "stwu 0,-16(1)\n\t" "mfmsr 5\n\t" "andi. 6,5,0xffef\n\t" "mtmsr 6\n\t" "isync\n\t" "sync\n\t" "lwz 3,0(3)\n\t" "mtmsr 5\n\t" "isync\n\t" "sync\n\t" "lwz 0,0(1)\n\t" "addi 1,1,16\n\t" "mtlr 0\n\t" "blr")
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:1952
__INTRIN_INLINE void _ReadWriteBarrier(void)
Definition: intrin_x86.h:88
#define _ReadBarrier
Definition: intrin_x86.h:95
__INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:1010
unsigned int uintptr_t
Definition: crtdefs.h:300
__INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:954
__INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long *const Addend, const long Value)
Definition: intrin_arm.h:359
__INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
Definition: intrin_x86.h:969
__INTRIN_INLINE void __writecr0(unsigned int Data)
Definition: intrin_x86.h:1718
static WCHAR Address[46]
Definition: ping.c:68
__INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
Definition: intrin_x86.h:1453
__INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile *Addend, char Value)
Definition: intrin_x86.h:436
__INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1228
__INTRIN_INLINE short _InterlockedDecrement16(volatile short *lpAddend)
Definition: intrin_x86.h:648
Definition: bufpool.h:45
__INTRIN_INLINE long _InterlockedCompareExchange(volatile long *Destination, long Exchange, long Comperand)
Definition: intrin_x86.h:382
_Must_inspect_result_ _In_ WDFKEY _In_ PCUNICODE_STRING _Out_opt_ PUSHORT _Inout_opt_ PUNICODE_STRING Value
Definition: wdfregistry.h:406
#define _WriteBarrier
Definition: intrin_x86.h:96
__INTRIN_INLINE void __nop(void)
Definition: intrin_x86.h:1971
__INTRIN_INLINE long _InterlockedAnd(volatile long *value, long mask)
Definition: intrin_x86.h:501
int Count
Definition: noreturn.cpp:7
__INTRIN_INLINE short _InterlockedAnd16(volatile short *value, short mask)
Definition: intrin_x86.h:482
__INTRIN_INLINE void _mm_pause(void)
Definition: intrin_x86.h:1965
GLboolean GLboolean GLboolean b
Definition: glext.h:6204
__INTRIN_INLINE long long __cdecl _abs64(long long value)
Definition: intrin_x86.h:1428
__INTRIN_INLINE void _mm_sfence(void)
Definition: intrin_x86.h:115
__INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
Definition: intrin_x86.h:1301
__INTRIN_INLINE char _InterlockedXor8(volatile char *value, char mask)
Definition: intrin_x86.h:577
__INTRIN_INLINE void * _InterlockedCompareExchangePointer(void *volatile *Destination, void *Exchange, void *Comperand)
Definition: intrin_x86.h:391
__INTRIN_INLINE long _InterlockedDecrement(volatile long *lpAddend)
Definition: intrin_x86.h:634
_In_ WDFCOLLECTION _In_ ULONG Index
__INTRIN_INLINE void * _InterlockedExchangePointer(void *volatile *Target, void *Value)
Definition: intrin_x86.h:427
__INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
Definition: intrin_x86.h:1219
__INTRIN_INLINE void __cdecl _enable(void)
Definition: intrin_x86.h:1638
__INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
Definition: intrin_x86.h:1918
GLuint GLuint num
Definition: glext.h:9618
__INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
Definition: intrin_x86.h:1573
_In_ PUNICODE_STRING _Inout_ PUNICODE_STRING Destination
Definition: rtlfuncs.h:2937
__INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
Definition: intrin_x86.h:1547
static int reg
Definition: i386-dis.c:1283
__INTRIN_INLINE char _InterlockedExchange8(volatile char *Target, char Value)
Definition: intrin_x86.h:400
__INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1532
__INTRIN_INLINE long _InterlockedExchangeAdd(volatile long *Addend, long Value)
Definition: intrin_x86.h:454
__INTRIN_INLINE void __int2c(void)
Definition: intrin_x86.h:1627
__INTRIN_INLINE long _InterlockedExchange(volatile long *Target, long Value)
Definition: intrin_x86.h:418
__INTRIN_INLINE short _InterlockedCompareExchange16(volatile short *Destination, short Exchange, short Comperand)
Definition: intrin_x86.h:373
#define __INTRIN_INLINE
Definition: intrin.h:39
__INTRIN_INLINE void __writeeflags(uintptr_t Value)
Definition: intrin_x86.h:1598
_In_ UCHAR _In_ UCHAR _In_ ULONG Code
Definition: wdfdevice.h:1697
__INTRIN_INLINE unsigned long long __rdtsc(void)
Definition: intrin_x86.h:1584
__INTRIN_INLINE void __cdecl _disable(void)
Definition: intrin_x86.h:1633
GLsizei const GLfloat * value
Definition: glext.h:6069
__INTRIN_INLINE unsigned int __readdr(unsigned int reg)
Definition: intrin_x86.h:1831
__INTRIN_INLINE char _InterlockedOr8(volatile char *value, char mask)
Definition: intrin_x86.h:520
__INTRIN_INLINE unsigned long __readcr0(void)
Definition: intrin_x86.h:1733
__INTRIN_INLINE unsigned long __readcr4(void)
Definition: intrin_x86.h:1754
processorSet Mask
__INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1485
__INTRIN_INLINE void _sgdt(void *Destination)
Definition: intrin_x86.h:1957
__INTRIN_INLINE unsigned long __indword(unsigned short Port)
Definition: intrin_x86.h:1467
__INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
Definition: intrin_x86.h:1420
__INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
Definition: intrin_x86.h:1263
#define local
Definition: zutil.h:30
_In_ ULONG _In_ ULONG Offset
Definition: ntddpcm.h:101
__INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
Definition: intrin_x86.h:1927
__INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
Definition: intrin_x86.h:1376
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1578
__INTRIN_INLINE char _InterlockedAnd8(volatile char *value, char mask)
Definition: intrin_x86.h:463
#define byte(x, n)
Definition: tomcrypt.h:118
__INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:1018
CPUINFO CPUInfo[]
Definition: parse.c:231
__INTRIN_INLINE void __incfsbyte(unsigned long Offset)
Definition: intrin_x86.h:986
__INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
Definition: intrin_x86.h:1558
__INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1287
void *__cdecl memmove(void *dest, const void *source, size_t num)
Definition: memmove.c:4
GLint GLint GLint GLint GLint GLint y
Definition: gl.h:1548
__INTRIN_INLINE short _InterlockedOr16(volatile short *value, short mask)
Definition: intrin_x86.h:539
const WCHAR * word
Definition: lex.c:36
_In_ WDFIOTARGET Target
Definition: wdfrequest.h:306
__INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
Definition: intrin_x86.h:1552
__INTRIN_INLINE short _InterlockedExchange16(volatile short *Target, short Value)
Definition: intrin_x86.h:409
__INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short *Addend, short Value)
Definition: intrin_x86.h:445
__INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1175
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1897
__INTRIN_INLINE unsigned char _bittest(const long *a, long b)
Definition: intrin_x86.h:1049
__INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1237
__INTRIN_INLINE short _InterlockedXor16(volatile short *value, short mask)
Definition: intrin_x86.h:596
__INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
Definition: intrin_x86.h:1329
__INTRIN_INLINE void _mm_mfence(void)
Definition: intrin_x86.h:99
__INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
Definition: intrin_x86.h:1564
_In_ UINT _In_ UINT _In_ PNDIS_PACKET Source
Definition: ndis.h:3167
static char * dest
Definition: rtl.c:135
__INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1527
GLboolean GLboolean GLboolean GLboolean a
Definition: glext.h:6204
__INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
Definition: intrin_x86.h:1512
__INTRIN_INLINE unsigned short __inword(unsigned short Port)
Definition: intrin_x86.h:1460
__INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
Definition: intrin_x86.h:1507
__INTRIN_INLINE void __movsw(unsigned short *Destination, const unsigned short *Source, size_t Count)
Definition: intrin_x86.h:816
__INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
Definition: intrin_x86.h:1517
__INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long *a, long b)
Definition: intrin_x86.h:742
__INTRIN_INLINE unsigned char _bittestandcomplement(long *a, long b)
Definition: intrin_x86.h:1093
__INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
Definition: intrin_x86.h:1390