ReactOS  0.4.14-dev-49-gfb4591c
intrin_x86.h
Go to the documentation of this file.
1 /*
2  Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3  Microsoft Visual C++ functions. Originally developed for the ReactOS
4  (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5  projects.
6 
7  Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8 
9  Permission is hereby granted, free of charge, to any person obtaining a
10  copy of this software and associated documentation files (the "Software"),
11  to deal in the Software without restriction, including without limitation
12  the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  and/or sell copies of the Software, and to permit persons to whom the
14  Software is furnished to do so, subject to the following conditions:
15 
16  The above copyright notice and this permission notice shall be included in
17  all copies or substantial portions of the Software.
18 
19  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  DEALINGS IN THE SOFTWARE.
26 */
27 
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30 
31 /*
32  FIXME: review all "memory" clobbers, add/remove to match Visual C++
33  behavior: some "obvious" memory barriers are not present in the Visual C++
34  implementation - e.g. __stosX; on the other hand, some memory barriers that
35  *are* present could have been missed
36 */
37 
38 /*
39  NOTE: this is a *compatibility* header. Some functions may look wrong at
40  first, but they're only "as wrong" as they would be on Visual C++. Our
41  priority is compatibility
42 
43  NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44  constraints and the uses of __volatile__ out of my... hat. Do not touch
45  them. I hate cargo cult programming
46 
47  NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48  barriers aren't there in Visual C++ (e.g. __stosX)
49 
50  NOTE: review all intrinsics with a return value, add/remove __volatile__
51  where necessary. If an intrinsic whose value is ignored generates a no-op
52  under Visual C++, __volatile__ must be omitted; if it always generates code
53  (for example, if it has side effects), __volatile__ must be specified. GCC
54  will only optimize out non-volatile asm blocks with outputs, so input-only
55  blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56  and follow Visual C++ behavior
57 
58  NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59  atomic operations. Test the version like this:
60 
61  #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62  ...
63 
64  Pay attention to the type of barrier. Make it match with what Visual C++
65  would use in the same case
66 */
67 
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71 
72 /*** memcopy must be memmove ***/
73 void* __cdecl memmove(void* dest, const void* source, size_t num);
74 __INTRIN_INLINE void* __cdecl memcpy(void* dest, const void* source, size_t num)
75 {
76  return memmove(dest, source, num);
77 }
78 
79 
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
84 
85 /*** Memory barriers ***/
86 
87 #if !HAS_BUILTIN(_ReadWriteBarrier)
89 {
90  __asm__ __volatile__("" : : : "memory");
91 }
92 #endif
93 
94 /* GCC only supports full barriers */
95 #define _ReadBarrier _ReadWriteBarrier
96 #define _WriteBarrier _ReadWriteBarrier
97 
98 #if !HAS_BUILTIN(_mm_mfence)
100 {
101  __asm__ __volatile__("mfence" : : : "memory");
102 }
103 #endif
104 
105 #if !HAS_BUILTIN(_mm_lfence)
107 {
108  _ReadBarrier();
109  __asm__ __volatile__("lfence");
110  _ReadBarrier();
111 }
112 #endif
113 
114 #if !HAS_BUILTIN(_mm_sfence)
116 {
117  _WriteBarrier();
118  __asm__ __volatile__("sfence");
119  _WriteBarrier();
120 }
121 #endif
122 
123 #ifdef __x86_64__
124 __INTRIN_INLINE void __faststorefence(void)
125 {
126  long local;
127  __asm__ __volatile__("lock; orl $0, %0;" : : "m"(local));
128 }
129 #endif
130 
131 
132 /*** Atomic operations ***/
133 
134 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
135 
136 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
137 {
138  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
139 }
140 
141 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
142 {
143  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
144 }
145 
146 #if !HAS_BUILTIN(_InterlockedCompareExchange)
147 __INTRIN_INLINE long __cdecl _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
148 {
149  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
150 }
151 #endif
152 
153 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
154 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
155 {
156  return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
157 }
158 #endif
159 
160 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
161 {
162  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
163  __sync_synchronize();
164  return __sync_lock_test_and_set(Target, Value);
165 }
166 
167 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
168 {
169  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
170  __sync_synchronize();
171  return __sync_lock_test_and_set(Target, Value);
172 }
173 
174 #if !HAS_BUILTIN(_InterlockedExchange)
175 __INTRIN_INLINE long __cdecl _InterlockedExchange(volatile long * Target, long Value)
176 {
177  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
178  __sync_synchronize();
179  return __sync_lock_test_and_set(Target, Value);
180 }
181 #endif
182 
183 #if !HAS_BUILTIN(_InterlockedExchangePointer)
184 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
185 {
186  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
187  __sync_synchronize();
188  return (void *)__sync_lock_test_and_set(Target, Value);
189 }
190 #endif
191 
192 #if defined(__x86_64__)
193 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * Target, long long Value)
194 {
195  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
196  __sync_synchronize();
197  return __sync_lock_test_and_set(Target, Value);
198 }
199 #endif
200 
201 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
202 {
203  return __sync_fetch_and_add(Addend, Value);
204 }
205 
206 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
207 {
208  return __sync_fetch_and_add(Addend, Value);
209 }
210 
211 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
212 __INTRIN_INLINE long __cdecl _InterlockedExchangeAdd(volatile long * Addend, long Value)
213 {
214  return __sync_fetch_and_add(Addend, Value);
215 }
216 #endif
217 
218 #if defined(__x86_64__)
219 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * Addend, long long Value)
220 {
221  return __sync_fetch_and_add(Addend, Value);
222 }
223 #endif
224 
225 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
226 {
227  return __sync_fetch_and_and(value, mask);
228 }
229 
230 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
231 {
232  return __sync_fetch_and_and(value, mask);
233 }
234 
235 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
236 {
237  return __sync_fetch_and_and(value, mask);
238 }
239 
240 #if defined(__x86_64__)
241 __INTRIN_INLINE long long _InterlockedAnd64(volatile long long * value, long long mask)
242 {
243  return __sync_fetch_and_and(value, mask);
244 }
245 #endif
246 
247 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
248 {
249  return __sync_fetch_and_or(value, mask);
250 }
251 
252 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
253 {
254  return __sync_fetch_and_or(value, mask);
255 }
256 
257 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
258 {
259  return __sync_fetch_and_or(value, mask);
260 }
261 
262 #if defined(__x86_64__)
263 __INTRIN_INLINE long long _InterlockedOr64(volatile long long * value, long long mask)
264 {
265  return __sync_fetch_and_or(value, mask);
266 }
267 #endif
268 
269 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
270 {
271  return __sync_fetch_and_xor(value, mask);
272 }
273 
274 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
275 {
276  return __sync_fetch_and_xor(value, mask);
277 }
278 
279 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
280 {
281  return __sync_fetch_and_xor(value, mask);
282 }
283 
284 #if defined(__x86_64__)
285 __INTRIN_INLINE long long _InterlockedXor64(volatile long long * value, long long mask)
286 {
287  return __sync_fetch_and_xor(value, mask);
288 }
289 #endif
290 
291 #if !HAS_BUILTIN(_InterlockedDecrement)
292 __INTRIN_INLINE long __cdecl _InterlockedDecrement(volatile long * lpAddend)
293 {
294  return __sync_sub_and_fetch(lpAddend, 1);
295 }
296 #endif
297 
298 #if !HAS_BUILTIN(_InterlockedIncrement)
299 __INTRIN_INLINE long __cdecl _InterlockedIncrement(volatile long * lpAddend)
300 {
301  return __sync_add_and_fetch(lpAddend, 1);
302 }
303 #endif
304 
305 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
306 {
307  return __sync_sub_and_fetch(lpAddend, 1);
308 }
309 
310 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
311 {
312  return __sync_add_and_fetch(lpAddend, 1);
313 }
314 
315 #if defined(__x86_64__)
316 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
317 {
318  return __sync_sub_and_fetch(lpAddend, 1);
319 }
320 
321 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
322 {
323  return __sync_add_and_fetch(lpAddend, 1);
324 }
325 #endif
326 
327 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
328 
329 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
330 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
331 {
332  char retval = Comperand;
333  __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
334  return retval;
335 }
336 #endif
337 
338 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
339 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
340 {
341  short retval = Comperand;
342  __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
343  return retval;
344 }
345 #endif
346 
347 #if !HAS_BUILTIN(_InterlockedCompareExchange)
348 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
349 {
350  long retval = Comperand;
351  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
352  return retval;
353 }
354 #endif
355 
356 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
357 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
358 {
359  void * retval = (void *)Comperand;
360  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
361  return retval;
362 }
363 #endif
364 
365 #if !HAS_BUILTIN(_InterlockedExchange8)
367 {
368  char retval = Value;
369  __asm__("xchgb %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
370  return retval;
371 }
372 #endif
373 
374 #if !HAS_BUILTIN(_InterlockedExchange16)
375 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
376 {
377  short retval = Value;
378  __asm__("xchgw %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
379  return retval;
380 }
381 #endif
382 
383 #if !HAS_BUILTIN(_InterlockedExchange)
385 {
386  long retval = Value;
387  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
388  return retval;
389 }
390 #endif
391 
392 #if !HAS_BUILTIN(_InterlockedExchangePointer)
393 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
394 {
395  void * retval = Value;
396  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
397  return retval;
398 }
399 #endif
400 
401 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
403 {
404  char retval = Value;
405  __asm__("lock; xaddb %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
406  return retval;
407 }
408 #endif
409 
410 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
412 {
413  short retval = Value;
414  __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
415  return retval;
416 }
417 #endif
418 
419 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
421 {
422  long retval = Value;
423  __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
424  return retval;
425 }
426 #endif
427 
428 #if !HAS_BUILTIN(_InterlockedAnd8)
429 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
430 {
431  char x;
432  char y;
433 
434  y = *value;
435 
436  do
437  {
438  x = y;
440  }
441  while(y != x);
442 
443  return y;
444 }
445 #endif
446 
447 #if !HAS_BUILTIN(_InterlockedAnd16)
448 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
449 {
450  short x;
451  short y;
452 
453  y = *value;
454 
455  do
456  {
457  x = y;
459  }
460  while(y != x);
461 
462  return y;
463 }
464 #endif
465 
466 #if !HAS_BUILTIN(_InterlockedAnd)
467 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
468 {
469  long x;
470  long y;
471 
472  y = *value;
473 
474  do
475  {
476  x = y;
478  }
479  while(y != x);
480 
481  return y;
482 }
483 #endif
484 
485 #if !HAS_BUILTIN(_InterlockedOr8)
486 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
487 {
488  char x;
489  char y;
490 
491  y = *value;
492 
493  do
494  {
495  x = y;
497  }
498  while(y != x);
499 
500  return y;
501 }
502 #endif
503 
504 #if !HAS_BUILTIN(_InterlockedOr16)
505 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
506 {
507  short x;
508  short y;
509 
510  y = *value;
511 
512  do
513  {
514  x = y;
516  }
517  while(y != x);
518 
519  return y;
520 }
521 #endif
522 
523 #if !HAS_BUILTIN(_InterlockedOr)
524 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
525 {
526  long x;
527  long y;
528 
529  y = *value;
530 
531  do
532  {
533  x = y;
535  }
536  while(y != x);
537 
538  return y;
539 }
540 #endif
541 
542 #if !HAS_BUILTIN(_InterlockedXor8)
543 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
544 {
545  char x;
546  char y;
547 
548  y = *value;
549 
550  do
551  {
552  x = y;
554  }
555  while(y != x);
556 
557  return y;
558 }
559 #endif
560 
561 #if !HAS_BUILTIN(_InterlockedXor16)
562 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
563 {
564  short x;
565  short y;
566 
567  y = *value;
568 
569  do
570  {
571  x = y;
573  }
574  while(y != x);
575 
576  return y;
577 }
578 #endif
579 
580 #if !HAS_BUILTIN(_InterlockedXor)
581 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
582 {
583  long x;
584  long y;
585 
586  y = *value;
587 
588  do
589  {
590  x = y;
592  }
593  while(y != x);
594 
595  return y;
596 }
597 #endif
598 
599 #if !HAS_BUILTIN(_InterlockedDecrement)
600 __INTRIN_INLINE long _InterlockedDecrement(volatile long * lpAddend)
601 {
602  return _InterlockedExchangeAdd(lpAddend, -1) - 1;
603 }
604 #endif
605 
606 #if !HAS_BUILTIN(_InterlockedIncrement)
607 __INTRIN_INLINE long _InterlockedIncrement(volatile long * lpAddend)
608 {
609  return _InterlockedExchangeAdd(lpAddend, 1) + 1;
610 }
611 #endif
612 
613 #if !HAS_BUILTIN(_InterlockedDecrement16)
614 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
615 {
616  return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
617 }
618 #endif
619 
620 #if !HAS_BUILTIN(_InterlockedIncrement16)
621 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
622 {
623  return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
624 }
625 #endif
626 
627 #if defined(__x86_64__)
628 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
629 {
630  return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
631 }
632 
633 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
634 {
635  return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
636 }
637 #endif
638 
639 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
640 
641 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
642 
643 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
644 {
645  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
646 }
647 
648 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
649 
650 #if !HAS_BUILTIN(_InterlockedCompareExchange64)
651 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
652 {
653  long long retval = Comperand;
654 
655  __asm__
656  (
657  "lock; cmpxchg8b %[Destination]" :
658  [retval] "+A" (retval) :
659  [Destination] "m" (*Destination),
660  "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
661  "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
662  "memory"
663  );
664 
665  return retval;
666 }
667 #endif
668 
669 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
670 
671 #ifdef __i386__
672 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * Addend, long Value)
673 {
674  __asm__
675  (
676  "lock; addl %[Value], %[Lo32];"
677  "jae LABEL%=;"
678  "lock; adcl $0, %[Hi32];"
679  "LABEL%=:;" :
680  [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
681  [Value] "ir" (Value) :
682  "memory"
683  );
684 
685  return Value;
686 }
687 #endif /* __i386__ */
688 
689 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, long b)
690 {
691  unsigned char retval;
692  __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
693  return retval;
694 }
695 
696 #if defined(__x86_64__)
697 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, long long b)
698 {
699  unsigned char retval;
700  __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
701  return retval;
702 }
703 #endif
704 
705 #if !HAS_BUILTIN(_interlockedbittestandset)
706 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, long b)
707 {
708  unsigned char retval;
709  __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
710  return retval;
711 }
712 #endif
713 
714 #if defined(__x86_64__)
715 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, long long b)
716 {
717  unsigned char retval;
718  __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
719  return retval;
720 }
721 #endif
722 
723 /*** String operations ***/
724 
725 #if !HAS_BUILTIN(__stosb)
726 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
727 __INTRIN_INLINE void __stosb(unsigned char * Dest, unsigned char Data, size_t Count)
728 {
729  __asm__ __volatile__
730  (
731  "rep; stosb" :
732  [Dest] "=D" (Dest), [Count] "=c" (Count) :
733  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
734  );
735 }
736 #endif
737 
738 __INTRIN_INLINE void __stosw(unsigned short * Dest, unsigned short Data, size_t Count)
739 {
740  __asm__ __volatile__
741  (
742  "rep; stosw" :
743  [Dest] "=D" (Dest), [Count] "=c" (Count) :
744  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
745  );
746 }
747 
748 __INTRIN_INLINE void __stosd(unsigned long * Dest, unsigned long Data, size_t Count)
749 {
750  __asm__ __volatile__
751  (
752  "rep; stosl" :
753  [Dest] "=D" (Dest), [Count] "=c" (Count) :
754  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
755  );
756 }
757 
758 #ifdef __x86_64__
759 __INTRIN_INLINE void __stosq(unsigned long long * Dest, unsigned long long Data, size_t Count)
760 {
761  __asm__ __volatile__
762  (
763  "rep; stosq" :
764  [Dest] "=D" (Dest), [Count] "=c" (Count) :
765  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
766  );
767 }
768 #endif
769 
770 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
771 {
772  __asm__ __volatile__
773  (
774  "rep; movsb" :
775  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
776  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
777  );
778 }
779 
780 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
781 {
782  __asm__ __volatile__
783  (
784  "rep; movsw" :
785  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
786  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
787  );
788 }
789 
790 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
791 {
792  __asm__ __volatile__
793  (
794  "rep; movsd" :
795  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
796  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
797  );
798 }
799 
800 #ifdef __x86_64__
801 __INTRIN_INLINE void __movsq(unsigned long * Destination, const unsigned long * Source, size_t Count)
802 {
803  __asm__ __volatile__
804  (
805  "rep; movsq" :
806  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
807  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
808  );
809 }
810 #endif
811 
812 #if defined(__x86_64__)
813 
814 /*** GS segment addressing ***/
815 
816 __INTRIN_INLINE void __writegsbyte(unsigned long Offset, unsigned char Data)
817 {
818  __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
819 }
820 
821 __INTRIN_INLINE void __writegsword(unsigned long Offset, unsigned short Data)
822 {
823  __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
824 }
825 
826 __INTRIN_INLINE void __writegsdword(unsigned long Offset, unsigned long Data)
827 {
828  __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
829 }
830 
831 __INTRIN_INLINE void __writegsqword(unsigned long Offset, unsigned long long Data)
832 {
833  __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
834 }
835 
836 __INTRIN_INLINE unsigned char __readgsbyte(unsigned long Offset)
837 {
838  unsigned char value;
839  __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
840  return value;
841 }
842 
843 __INTRIN_INLINE unsigned short __readgsword(unsigned long Offset)
844 {
845  unsigned short value;
846  __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
847  return value;
848 }
849 
850 __INTRIN_INLINE unsigned long __readgsdword(unsigned long Offset)
851 {
852  unsigned long value;
853  __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
854  return value;
855 }
856 
857 __INTRIN_INLINE unsigned long long __readgsqword(unsigned long Offset)
858 {
859  unsigned long long value;
860  __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
861  return value;
862 }
863 
864 __INTRIN_INLINE void __incgsbyte(unsigned long Offset)
865 {
866  __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
867 }
868 
869 __INTRIN_INLINE void __incgsword(unsigned long Offset)
870 {
871  __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
872 }
873 
874 __INTRIN_INLINE void __incgsdword(unsigned long Offset)
875 {
876  __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
877 }
878 
879 __INTRIN_INLINE void __incgsqword(unsigned long Offset)
880 {
881  __asm__ __volatile__("incq %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
882 }
883 
884 __INTRIN_INLINE void __addgsbyte(unsigned long Offset, unsigned char Data)
885 {
886  __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
887 }
888 
889 __INTRIN_INLINE void __addgsword(unsigned long Offset, unsigned short Data)
890 {
891  __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
892 }
893 
894 __INTRIN_INLINE void __addgsdword(unsigned long Offset, unsigned int Data)
895 {
896  __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
897 }
898 
899 __INTRIN_INLINE void __addgsqword(unsigned long Offset, unsigned long long Data)
900 {
901  __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
902 }
903 
904 #else /* defined(__x86_64__) */
905 
906 /*** FS segment addressing ***/
907 
908 __INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
909 {
910  __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
911 }
912 
913 __INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
914 {
915  __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
916 }
917 
918 __INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
919 {
920  __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
921 }
922 
923 #if !HAS_BUILTIN(__readfsbyte)
924 __INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
925 {
926  unsigned char value;
927  __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
928  return value;
929 }
930 #endif
931 
932 #if !HAS_BUILTIN(__readfsword)
933 __INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
934 {
935  unsigned short value;
936  __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
937  return value;
938 }
939 #endif
940 
941 #if !HAS_BUILTIN(__readfsdword)
942 __INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
943 {
944  unsigned long value;
945  __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
946  return value;
947 }
948 #endif
949 
950 __INTRIN_INLINE void __incfsbyte(unsigned long Offset)
951 {
952  __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
953 }
954 
955 __INTRIN_INLINE void __incfsword(unsigned long Offset)
956 {
957  __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
958 }
959 
961 {
962  __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
963 }
964 
965 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
966 __INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
967 {
968  if(!__builtin_constant_p(Offset))
969  __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
970  else
971  __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
972 }
973 
974 __INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
975 {
976  if(!__builtin_constant_p(Offset))
977  __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
978  else
979  __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
980 }
981 
982 __INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
983 {
984  if(!__builtin_constant_p(Offset))
985  __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
986  else
987  __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
988 }
989 
990 #endif /* defined(__x86_64__) */
991 
992 
993 /*** Bit manipulation ***/
994 
995 #if !HAS_BUILTIN(_BitScanForward)
996 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * Index, unsigned long Mask)
997 {
998  __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
999  return Mask ? 1 : 0;
1000 }
1001 #endif
1002 
1003 #if !HAS_BUILTIN(_BitScanReverse)
1004 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * Index, unsigned long Mask)
1005 {
1006  __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1007  return Mask ? 1 : 0;
1008 }
1009 #endif
1010 
1011 /* NOTE: again, the bizarre implementation follows Visual C++ */
1012 __INTRIN_INLINE unsigned char _bittest(const long * a, long b)
1013 {
1014  unsigned char retval;
1015 
1016  if(__builtin_constant_p(b))
1017  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
1018  else
1019  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1020 
1021  return retval;
1022 }
1023 
1024 #ifdef __x86_64__
1025 __INTRIN_INLINE unsigned char _BitScanForward64(unsigned long * Index, unsigned long long Mask)
1026 {
1027  unsigned long long Index64;
1028  __asm__("bsfq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1029  *Index = Index64;
1030  return Mask ? 1 : 0;
1031 }
1032 
1033 __INTRIN_INLINE unsigned char _BitScanReverse64(unsigned long * Index, unsigned long long Mask)
1034 {
1035  unsigned long long Index64;
1036  __asm__("bsrq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1037  *Index = Index64;
1038  return Mask ? 1 : 0;
1039 }
1040 
1041 __INTRIN_INLINE unsigned char _bittest64(const long long * a, long long b)
1042 {
1043  unsigned char retval;
1044 
1045  if(__builtin_constant_p(b))
1046  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
1047  else
1048  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1049 
1050  return retval;
1051 }
1052 #endif
1053 
1054 __INTRIN_INLINE unsigned char _bittestandcomplement(long * a, long b)
1055 {
1056  unsigned char retval;
1057 
1058  if(__builtin_constant_p(b))
1059  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1060  else
1061  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1062 
1063  return retval;
1064 }
1065 
1066 __INTRIN_INLINE unsigned char _bittestandreset(long * a, long b)
1067 {
1068  unsigned char retval;
1069 
1070  if(__builtin_constant_p(b))
1071  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1072  else
1073  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1074 
1075  return retval;
1076 }
1077 
1078 __INTRIN_INLINE unsigned char _bittestandset(long * a, long b)
1079 {
1080  unsigned char retval;
1081 
1082  if(__builtin_constant_p(b))
1083  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1084  else
1085  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1086 
1087  return retval;
1088 }
1089 
1090 #ifdef __x86_64__
1091 
1092 __INTRIN_INLINE unsigned char _bittestandset64(long long * a, long long b)
1093 {
1094  unsigned char retval;
1095 
1096  if(__builtin_constant_p(b))
1097  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1098  else
1099  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1100 
1101  return retval;
1102 }
1103 
1104 __INTRIN_INLINE unsigned char _bittestandreset64(long long * a, long long b)
1105 {
1106  unsigned char retval;
1107 
1108  if(__builtin_constant_p(b))
1109  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1110  else
1111  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1112 
1113  return retval;
1114 }
1115 
1116 __INTRIN_INLINE unsigned char _bittestandcomplement64(long long * a, long long b)
1117 {
1118  unsigned char retval;
1119 
1120  if(__builtin_constant_p(b))
1121  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1122  else
1123  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1124 
1125  return retval;
1126 }
1127 
1128 #endif /* __x86_64__ */
1129 
1130 #if !HAS_BUILTIN(_rotl8)
1131 __INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
1132 {
1133  unsigned char retval;
1134  __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1135  return retval;
1136 }
1137 #endif
1138 
1139 #if !HAS_BUILTIN(_rotl16)
1140 __INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
1141 {
1142  unsigned short retval;
1143  __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1144  return retval;
1145 }
1146 #endif
1147 
1148 #if !HAS_BUILTIN(_rotl)
1149 __INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
1150 {
1151  unsigned int retval;
1152  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1153  return retval;
1154 }
1155 #endif
1156 
1157 #ifdef __x86_64__
1158 __INTRIN_INLINE unsigned long long _rotl64(unsigned long long value, int shift)
1159 {
1160  unsigned long long retval;
1161  __asm__("rolq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1162  return retval;
1163 }
1164 #else /* __x86_64__ */
1165 #if !HAS_BUILTIN(_rotl64)
1166 __INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
1167 {
1168  /* FIXME: this is probably not optimal */
1169  return (value << shift) | (value >> (64 - shift));
1170 }
1171 #endif /* !HAS_BUILTIN(_rotl64) */
1172 #endif /* __x86_64__ */
1173 
1174 #if !HAS_BUILTIN(_rotr)
1175 __INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
1176 {
1177  unsigned int retval;
1178  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1179  return retval;
1180 }
1181 #endif
1182 
1183 #if !HAS_BUILTIN(_rotr8)
1184 __INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
1185 {
1186  unsigned char retval;
1187  __asm__("rorb %b[shift], %b[retval]" : [retval] "=qm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1188  return retval;
1189 }
1190 #endif
1191 
1192 #if !HAS_BUILTIN(_rotr16)
1193 __INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
1194 {
1195  unsigned short retval;
1196  __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1197  return retval;
1198 }
1199 #endif
1200 
1201 #ifdef __x86_64__
1202 __INTRIN_INLINE unsigned long long _rotr64(unsigned long long value, int shift)
1203 {
1204  unsigned long long retval;
1205  __asm__("rorq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1206  return retval;
1207 }
1208 #else /* __x86_64__ */
1209 #if !HAS_BUILTIN(_rotr64)
1210 __INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
1211 {
1212  /* FIXME: this is probably not optimal */
1213  return (value >> shift) | (value << (64 - shift));
1214 }
1215 #endif /* !HAS_BUILTIN(_rotr64) */
1216 #endif /* __x86_64__ */
1217 
1218 #if !HAS_BUILTIN(_lrotl)
1219 __INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
1220 {
1221  unsigned long retval;
1222  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1223  return retval;
1224 }
1225 #endif
1226 
1227 #if !HAS_BUILTIN(_lrotr)
1228 __INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
1229 {
1230  unsigned long retval;
1231  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1232  return retval;
1233 }
1234 #endif
1235 
1236 /*
1237  NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1238  constraint (edx:eax) for the Mask argument, because it's the only way GCC
1239  can pass 64-bit operands around - passing the two 32 bit parts separately
1240  just confuses it. Also we declare Bit as an int and then truncate it to
1241  match Visual C++ behavior
1242 */
1243 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1244 {
1245  unsigned long long retval = Mask;
1246 
1247  __asm__
1248  (
1249  "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1250  "+A" (retval) :
1251  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1252  );
1253 
1254  return retval;
1255 }
1256 
1257 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1258 {
1259  long long retval = Mask;
1260 
1261  __asm__
1262  (
1263  "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1264  "+A" (retval) :
1265  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1266  );
1267 
1268  return retval;
1269 }
1270 
1271 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1272 {
1273  unsigned long long retval = Mask;
1274 
1275  __asm__
1276  (
1277  "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1278  "+A" (retval) :
1279  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1280  );
1281 
1282  return retval;
1283 }
1284 
1285 __INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
1286 {
1287  unsigned short retval;
1288  __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
1289  return retval;
1290 }
1291 
1292 __INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
1293 {
1294  unsigned long retval;
1295  __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1296  return retval;
1297 }
1298 
1299 #ifdef __x86_64__
1300 __INTRIN_INLINE unsigned long long _byteswap_uint64(unsigned long long value)
1301 {
1302  unsigned long long retval;
1303  __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1304  return retval;
1305 }
1306 #else
1307 __INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
1308 {
1309  union {
1310  unsigned long long int64part;
1311  struct {
1312  unsigned long lowpart;
1313  unsigned long hipart;
1314  };
1315  } retval;
1316  retval.int64part = value;
1317  __asm__("bswapl %[lowpart]\n"
1318  "bswapl %[hipart]\n"
1319  : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
1320  return retval.int64part;
1321 }
1322 #endif
1323 
1324 __INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
1325 {
1326  return __builtin_clz(value);
1327 }
1328 
1329 __INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
1330 {
1331  return __builtin_clz(value);
1332 }
1333 
1334 #if !HAS_BUILTIN(__popcnt)
1335 __INTRIN_INLINE unsigned int __popcnt(unsigned int value)
1336 {
1337  return __builtin_popcount(value);
1338 }
1339 #endif
1340 
1341 #if !HAS_BUILTIN(__popcnt16)
1342 __INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
1343 {
1344  return __builtin_popcount(value);
1345 }
1346 #endif
1347 
1348 #ifdef __x86_64__
1349 __INTRIN_INLINE unsigned long long __lzcnt64(unsigned long long value)
1350 {
1351  return __builtin_clzll(value);
1352 }
1353 
1354 __INTRIN_INLINE unsigned long long __popcnt64(unsigned long long value)
1355 {
1356  return __builtin_popcountll(value);
1357 }
1358 #endif
1359 
1360 /*** 64-bit math ***/
1361 
1362 #if !HAS_BUILTIN(__emul)
1363 __INTRIN_INLINE long long __emul(int a, int b)
1364 {
1365  long long retval;
1366  __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1367  return retval;
1368 }
1369 #endif
1370 
1371 #if !HAS_BUILTIN(__emulu)
1372 __INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
1373 {
1374  unsigned long long retval;
1375  __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1376  return retval;
1377 }
1378 #endif
1379 
1380 __INTRIN_INLINE long long __cdecl _abs64(long long value)
1381 {
1382  return (value >= 0) ? value : -value;
1383 }
1384 
1385 #ifdef __x86_64__
1386 
1387 __INTRIN_INLINE long long __mulh(long long a, long long b)
1388 {
1389  long long retval;
1390  __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1391  return retval;
1392 }
1393 
1394 __INTRIN_INLINE unsigned long long __umulh(unsigned long long a, unsigned long long b)
1395 {
1396  unsigned long long retval;
1397  __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1398  return retval;
1399 }
1400 
1401 #endif
1402 
1403 /*** Port I/O ***/
1404 
1405 __INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
1406 {
1407  unsigned char byte;
1408  __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1409  return byte;
1410 }
1411 
1412 __INTRIN_INLINE unsigned short __inword(unsigned short Port)
1413 {
1414  unsigned short word;
1415  __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1416  return word;
1417 }
1418 
1419 __INTRIN_INLINE unsigned long __indword(unsigned short Port)
1420 {
1421  unsigned long dword;
1422  __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1423  return dword;
1424 }
1425 
1426 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1427 {
1428  __asm__ __volatile__
1429  (
1430  "rep; insb" :
1431  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1432  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1433  "memory"
1434  );
1435 }
1436 
1437 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1438 {
1439  __asm__ __volatile__
1440  (
1441  "rep; insw" :
1442  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1443  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1444  "memory"
1445  );
1446 }
1447 
1448 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1449 {
1450  __asm__ __volatile__
1451  (
1452  "rep; insl" :
1453  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1454  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1455  "memory"
1456  );
1457 }
1458 
1459 __INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
1460 {
1461  __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1462 }
1463 
1464 __INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
1465 {
1466  __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1467 }
1468 
1469 __INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
1470 {
1471  __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1472 }
1473 
1474 __INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1475 {
1476  __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1477 }
1478 
1479 __INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1480 {
1481  __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1482 }
1483 
1484 __INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1485 {
1486  __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1487 }
1488 
1489 __INTRIN_INLINE int __cdecl _inp(unsigned short Port)
1490 {
1491  return __inbyte(Port);
1492 }
1493 
1494 __INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
1495 {
1496  return __inword(Port);
1497 }
1498 
1499 __INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
1500 {
1501  return __indword(Port);
1502 }
1503 
1504 __INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
1505 {
1506  __outbyte(Port, (unsigned char)databyte);
1507  return databyte;
1508 }
1509 
1510 __INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
1511 {
1512  __outword(Port, dataword);
1513  return dataword;
1514 }
1515 
1516 __INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
1517 {
1518  __outdword(Port, dataword);
1519  return dataword;
1520 }
1521 
1522 
1523 /*** System information ***/
1524 
1525 __INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
1526 {
1527  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1528 }
1529 
1530 __INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
1531 {
1532  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType), "c" (ECXValue));
1533 }
1534 
1535 #if !HAS_BUILTIN(__rdtsc)
1536 __INTRIN_INLINE unsigned long long __rdtsc(void)
1537 {
1538 #ifdef __x86_64__
1539  unsigned long long low, high;
1540  __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1541  return low | (high << 32);
1542 #else
1543  unsigned long long retval;
1544  __asm__ __volatile__("rdtsc" : "=A"(retval));
1545  return retval;
1546 #endif
1547 }
1548 #endif /* !HAS_BUILTIN(__rdtsc) */
1549 
1551 {
1552  __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1553 }
1554 
1556 {
1557  uintptr_t retval;
1558  __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1559  return retval;
1560 }
1561 
1562 /*** Interrupts ***/
1563 
1564 #if !HAS_BUILTIN(__debugbreak)
1566 {
1567  __asm__("int $3");
1568 }
1569 #endif
1570 
1571 #if !HAS_BUILTIN(__ud2)
1573 {
1574  __asm__("ud2");
1575 }
1576 #endif
1577 
1578 #if !HAS_BUILTIN(__int2c)
1580 {
1581  __asm__("int $0x2c");
1582 }
1583 #endif
1584 
1586 {
1587  __asm__("cli" : : : "memory");
1588 }
1589 
1591 {
1592  __asm__("sti" : : : "memory");
1593 }
1594 
1596 {
1597  __asm__("hlt" : : : "memory");
1598 }
1599 
1600 #if !HAS_BUILTIN(__fastfail)
1601 __declspec(noreturn)
1602 __INTRIN_INLINE void __fastfail(unsigned int Code)
1603 {
1604  __asm__("int $0x29" : : "c"(Code) : "memory");
1605  __builtin_unreachable();
1606 }
1607 #endif
1608 
1609 /*** Protected memory management ***/
1610 
1611 #ifdef __x86_64__
1612 
1613 __INTRIN_INLINE void __writecr0(unsigned long long Data)
1614 {
1615  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1616 }
1617 
1618 __INTRIN_INLINE void __writecr3(unsigned long long Data)
1619 {
1620  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1621 }
1622 
1623 __INTRIN_INLINE void __writecr4(unsigned long long Data)
1624 {
1625  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1626 }
1627 
1628 __INTRIN_INLINE void __writecr8(unsigned long long Data)
1629 {
1630  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1631 }
1632 
1633 __INTRIN_INLINE unsigned long long __readcr0(void)
1634 {
1635  unsigned long long value;
1636  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1637  return value;
1638 }
1639 
1640 __INTRIN_INLINE unsigned long long __readcr2(void)
1641 {
1642  unsigned long long value;
1643  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1644  return value;
1645 }
1646 
1647 __INTRIN_INLINE unsigned long long __readcr3(void)
1648 {
1649  unsigned long long value;
1650  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1651  return value;
1652 }
1653 
1654 __INTRIN_INLINE unsigned long long __readcr4(void)
1655 {
1656  unsigned long long value;
1657  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1658  return value;
1659 }
1660 
1661 __INTRIN_INLINE unsigned long long __readcr8(void)
1662 {
1663  unsigned long long value;
1664  __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1665  return value;
1666 }
1667 
1668 #else /* __x86_64__ */
1669 
1670 __INTRIN_INLINE void __writecr0(unsigned int Data)
1671 {
1672  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1673 }
1674 
1675 __INTRIN_INLINE void __writecr3(unsigned int Data)
1676 {
1677  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1678 }
1679 
1680 __INTRIN_INLINE void __writecr4(unsigned int Data)
1681 {
1682  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1683 }
1684 
1685 #if !HAS_BUILTIN(__writecr8)
1686 __INTRIN_INLINE void __writecr8(unsigned int Data)
1687 {
1688  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1689 }
1690 #endif
1691 
1692 __INTRIN_INLINE unsigned long __readcr0(void)
1693 {
1694  unsigned long value;
1695  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1696  return value;
1697 }
1698 
1699 __INTRIN_INLINE unsigned long __readcr2(void)
1700 {
1701  unsigned long value;
1702  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1703  return value;
1704 }
1705 
1706 __INTRIN_INLINE unsigned long __readcr3(void)
1707 {
1708  unsigned long value;
1709  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1710  return value;
1711 }
1712 
1713 __INTRIN_INLINE unsigned long __readcr4(void)
1714 {
1715  unsigned long value;
1716  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1717  return value;
1718 }
1719 
1720 #if !HAS_BUILTIN(__readcr8)
1721 __INTRIN_INLINE unsigned long __readcr8(void)
1722 {
1723  unsigned long value;
1724  __asm__ __volatile__("mov %%cr8, %[value]" : [value] "=r" (value));
1725  return value;
1726 }
1727 #endif
1728 
1729 #endif /* __x86_64__ */
1730 
1731 #ifdef __x86_64__
1732 
1733 __INTRIN_INLINE unsigned long long __readdr(unsigned int reg)
1734 {
1735  unsigned long long value;
1736  switch (reg)
1737  {
1738  case 0:
1739  __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1740  break;
1741  case 1:
1742  __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1743  break;
1744  case 2:
1745  __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1746  break;
1747  case 3:
1748  __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1749  break;
1750  case 4:
1751  __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1752  break;
1753  case 5:
1754  __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1755  break;
1756  case 6:
1757  __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1758  break;
1759  case 7:
1760  __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1761  break;
1762  }
1763  return value;
1764 }
1765 
1766 __INTRIN_INLINE void __writedr(unsigned reg, unsigned long long value)
1767 {
1768  switch (reg)
1769  {
1770  case 0:
1771  __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1772  break;
1773  case 1:
1774  __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1775  break;
1776  case 2:
1777  __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1778  break;
1779  case 3:
1780  __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1781  break;
1782  case 4:
1783  __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1784  break;
1785  case 5:
1786  __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1787  break;
1788  case 6:
1789  __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1790  break;
1791  case 7:
1792  __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1793  break;
1794  }
1795 }
1796 
1797 #else /* __x86_64__ */
1798 
1799 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1800 {
1801  unsigned int value;
1802  switch (reg)
1803  {
1804  case 0:
1805  __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1806  break;
1807  case 1:
1808  __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1809  break;
1810  case 2:
1811  __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1812  break;
1813  case 3:
1814  __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1815  break;
1816  case 4:
1817  __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1818  break;
1819  case 5:
1820  __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1821  break;
1822  case 6:
1823  __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1824  break;
1825  case 7:
1826  __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1827  break;
1828  }
1829  return value;
1830 }
1831 
1832 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1833 {
1834  switch (reg)
1835  {
1836  case 0:
1837  __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1838  break;
1839  case 1:
1840  __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1841  break;
1842  case 2:
1843  __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1844  break;
1845  case 3:
1846  __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1847  break;
1848  case 4:
1849  __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1850  break;
1851  case 5:
1852  __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1853  break;
1854  case 6:
1855  __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1856  break;
1857  case 7:
1858  __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1859  break;
1860  }
1861 }
1862 
1863 #endif /* __x86_64__ */
1864 
1866 {
1867  __asm__ __volatile__ ("invlpg (%[Address])" : : [Address] "b" (Address) : "memory");
1868 }
1869 
1870 
1871 /*** System operations ***/
1872 
1873 __INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
1874 {
1875 #ifdef __x86_64__
1876  unsigned long low, high;
1877  __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1878  return ((unsigned long long)high << 32) | low;
1879 #else
1880  unsigned long long retval;
1881  __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1882  return retval;
1883 #endif
1884 }
1885 
1886 __INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
1887 {
1888 #ifdef __x86_64__
1889  __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1890 #else
1891  __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1892 #endif
1893 }
1894 
1895 __INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
1896 {
1897  unsigned long long retval;
1898  __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
1899  return retval;
1900 }
1901 
1902 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1903 __INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
1904 {
1905  unsigned long retval;
1906  __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
1907  return retval;
1908 }
1909 
1911 {
1912  __asm__ __volatile__("wbinvd" : : : "memory");
1913 }
1914 
1916 {
1917  __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
1918 }
1919 
1921 {
1922  __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
1923 }
1924 
1926 {
1927  __asm__ __volatile__("sgdt %0" : : "m"(*(short*)Destination) : "memory");
1928 }
1929 
1930 /*** Misc operations ***/
1931 
1932 #if !HAS_BUILTIN(_mm_pause)
1934 {
1935  __asm__ __volatile__("pause" : : : "memory");
1936 }
1937 #endif
1938 
1940 {
1941  __asm__ __volatile__("nop");
1942 }
1943 
1944 #ifdef __cplusplus
1945 }
1946 #endif
1947 
1948 #endif /* KJK_INTRIN_X86_H_ */
1949 
1950 /* EOF */
__INTRIN_INLINE void __ud2(void)
Definition: intrin_x86.h:1572
_In_opt_ ULONG _Out_ PULONG Value
Definition: rtlfuncs.h:2343
CPPORT Port[4]
Definition: headless.c:34
__INTRIN_INLINE void __writecr4(unsigned int Data)
Definition: intrin_x86.h:1680
__INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
Definition: intrin_x86.h:1494
__INTRIN_INLINE void __stosd(unsigned long *Dest, unsigned long Data, size_t Count)
Definition: intrin_x86.h:748
__INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
Definition: intrin_x86.h:1832
__INTRIN_INLINE void _mm_lfence(void)
Definition: intrin_x86.h:106
__INTRIN_INLINE void __stosb(unsigned char *Dest, unsigned char Data, size_t Count)
Definition: intrin_x86.h:727
__INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
Definition: intrin_x86.h:1166
#define shift
Definition: input.c:1668
__INTRIN_INLINE unsigned long __readcr8(void)
Definition: intrin_x86.h:1721
__INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:966
__INTRIN_INLINE void *__cdecl memcpy(void *dest, const void *source, size_t num)
Definition: intrin_x86.h:74
__INTRIN_INLINE char _InterlockedCompareExchange8(volatile char *Destination, char Exchange, char Comperand)
Definition: intrin_x86.h:330
__INTRIN_INLINE unsigned long __readcr2(void)
Definition: intrin_x86.h:1699
__INTRIN_INLINE void __movsb(unsigned char *Destination, const unsigned char *Source, size_t Count)
Definition: intrin_x86.h:770
__INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
Definition: intrin_x86.h:1873
__INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long *a, long b)
Definition: intrin_x86.h:689
#define __cdecl
Definition: accygwin.h:79
__INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:908
__INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1140
__INTRIN_INLINE void __incfsdword(unsigned long Offset)
Definition: intrin_x86.h:960
__INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
Definition: intrin_x86.h:942
__INTRIN_INLINE unsigned char _BitScanReverse(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1004
__INTRIN_INLINE void __wbinvd(void)
Definition: intrin_x86.h:1910
__INTRIN_INLINE long _InterlockedXor(volatile long *value, long mask)
Definition: intrin_x86.h:581
__INTRIN_INLINE void __incfsword(unsigned long Offset)
Definition: intrin_x86.h:955
__INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
Definition: intrin_x86.h:1228
__INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1426
__INTRIN_INLINE void __halt(void)
Definition: intrin_x86.h:1595
__INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long *Destination, long long Exchange, long long Comperand)
Definition: intrin_x86.h:651
__INTRIN_INLINE void __stosw(unsigned short *Dest, unsigned short Data, size_t Count)
Definition: intrin_x86.h:738
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1015
__INTRIN_INLINE void __lidt(void *Source)
Definition: intrin_x86.h:1915
__INTRIN_INLINE unsigned int __popcnt(unsigned int value)
Definition: intrin_x86.h:1335
GLint GLint GLint GLint GLint x
Definition: gl.h:1548
__INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1271
__INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
Definition: intrin_x86.h:1903
__declspec(noreturn) __INTRIN_INLINE void __fastfail(unsigned int Code)
Definition: intrin_x86.h:1601
__INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
Definition: intrin_x86.h:1307
__INTRIN_INLINE unsigned long __readcr3(void)
Definition: intrin_x86.h:1706
__INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
Definition: intrin_x86.h:924
__INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1448
__INTRIN_INLINE void __movsd(unsigned long *Destination, const unsigned long *Source, size_t Count)
Definition: intrin_x86.h:790
__INTRIN_INLINE long long __emul(int a, int b)
Definition: intrin_x86.h:1363
IN OUT PLONG Addend
Definition: CrNtStubs.h:22
__INTRIN_INLINE void __writecr8(unsigned int Data)
Definition: intrin_x86.h:1686
__INTRIN_INLINE short _InterlockedIncrement16(volatile short *lpAddend)
Definition: intrin_x86.h:621
__INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:913
__INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
Definition: intrin_x86.h:1149
__INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
Definition: intrin_x86.h:1210
__INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
Definition: intrin_x86.h:1324
__INTRIN_INLINE unsigned char _BitScanForward(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:996
__INTRIN_INLINE unsigned char _bittestandset(long *a, long b)
Definition: intrin_x86.h:1078
__INTRIN_INLINE long _InterlockedOr(volatile long *value, long mask)
Definition: intrin_x86.h:524
__INTRIN_INLINE int __cdecl _inp(unsigned short Port)
Definition: intrin_x86.h:1489
__INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
Definition: intrin_x86.h:1292
__INTRIN_INLINE void __cdecl __debugbreak(void)
Definition: intrin_x86.h:1565
__INTRIN_INLINE void __writecr3(unsigned int Data)
Definition: intrin_x86.h:1675
__INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1474
__INTRIN_INLINE unsigned char _bittestandreset(long *a, long b)
Definition: intrin_x86.h:1066
GLenum GLint GLuint mask
Definition: glext.h:6028
__INTRIN_INLINE uintptr_t __readeflags(void)
Definition: intrin_x86.h:1555
_In_ ULONG _In_ ULONG Offset
Definition: ntddpcm.h:101
__INTRIN_INLINE long _InterlockedIncrement(volatile long *lpAddend)
Definition: intrin_x86.h:607
__asm__("\t.globl GetPhys\n" "GetPhys:\t\n" "mflr 0\n\t" "stwu 0,-16(1)\n\t" "mfmsr 5\n\t" "andi. 6,5,0xffef\n\t" "mtmsr 6\n\t" "isync\n\t" "sync\n\t" "lwz 3,0(3)\n\t" "mtmsr 5\n\t" "isync\n\t" "sync\n\t" "lwz 0,0(1)\n\t" "addi 1,1,16\n\t" "mtlr 0\n\t" "blr")
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:1920
__INTRIN_INLINE void _ReadWriteBarrier(void)
Definition: intrin_x86.h:88
#define _ReadBarrier
Definition: intrin_x86.h:95
__INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:974
unsigned int uintptr_t
Definition: crtdefs.h:300
__INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:918
__INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long *const Addend, const long Value)
Definition: intrin_arm.h:359
__INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
Definition: intrin_x86.h:933
__INTRIN_INLINE void __writecr0(unsigned int Data)
Definition: intrin_x86.h:1670
static WCHAR Address[46]
Definition: ping.c:68
__INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
Definition: intrin_x86.h:1405
__INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile *Addend, char Value)
Definition: intrin_x86.h:402
__INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1184
__INTRIN_INLINE short _InterlockedDecrement16(volatile short *lpAddend)
Definition: intrin_x86.h:614
Definition: bufpool.h:45
__INTRIN_INLINE long _InterlockedCompareExchange(volatile long *Destination, long Exchange, long Comperand)
Definition: intrin_x86.h:348
#define _WriteBarrier
Definition: intrin_x86.h:96
#define Code
Definition: deflate.h:80
__INTRIN_INLINE void __nop(void)
Definition: intrin_x86.h:1939
__INTRIN_INLINE long _InterlockedAnd(volatile long *value, long mask)
Definition: intrin_x86.h:467
__INTRIN_INLINE short _InterlockedAnd16(volatile short *value, short mask)
Definition: intrin_x86.h:448
__INTRIN_INLINE void _mm_pause(void)
Definition: intrin_x86.h:1933
GLboolean GLboolean GLboolean b
Definition: glext.h:6204
__INTRIN_INLINE long long __cdecl _abs64(long long value)
Definition: intrin_x86.h:1380
__INTRIN_INLINE void _mm_sfence(void)
Definition: intrin_x86.h:115
__INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
Definition: intrin_x86.h:1257
__INTRIN_INLINE char _InterlockedXor8(volatile char *value, char mask)
Definition: intrin_x86.h:543
__INTRIN_INLINE void * _InterlockedCompareExchangePointer(void *volatile *Destination, void *Exchange, void *Comperand)
Definition: intrin_x86.h:357
__INTRIN_INLINE long _InterlockedDecrement(volatile long *lpAddend)
Definition: intrin_x86.h:600
__INTRIN_INLINE void * _InterlockedExchangePointer(void *volatile *Target, void *Value)
Definition: intrin_x86.h:393
__INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
Definition: intrin_x86.h:1175
static const UCHAR Index[8]
Definition: usbohci.c:18
__INTRIN_INLINE void __cdecl _enable(void)
Definition: intrin_x86.h:1590
__INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
Definition: intrin_x86.h:1886
GLuint GLuint num
Definition: glext.h:9618
__INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
Definition: intrin_x86.h:1525
_In_ PUNICODE_STRING _Inout_ PUNICODE_STRING Destination
Definition: rtlfuncs.h:2891
__INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
Definition: intrin_x86.h:1499
_Must_inspect_result_ typedef _In_ ULONG _In_ BOOLEAN Target
Definition: iotypes.h:1068
__INTRIN_INLINE char _InterlockedExchange8(volatile char *Target, char Value)
Definition: intrin_x86.h:366
__INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1484
__INTRIN_INLINE long _InterlockedExchangeAdd(volatile long *Addend, long Value)
Definition: intrin_x86.h:420
__INTRIN_INLINE void __int2c(void)
Definition: intrin_x86.h:1579
__INTRIN_INLINE long _InterlockedExchange(volatile long *Target, long Value)
Definition: intrin_x86.h:384
__INTRIN_INLINE short _InterlockedCompareExchange16(volatile short *Destination, short Exchange, short Comperand)
Definition: intrin_x86.h:339
#define __INTRIN_INLINE
Definition: intrin.h:39
__INTRIN_INLINE void __writeeflags(uintptr_t Value)
Definition: intrin_x86.h:1550
__INTRIN_INLINE unsigned long long __rdtsc(void)
Definition: intrin_x86.h:1536
__INTRIN_INLINE void __cdecl _disable(void)
Definition: intrin_x86.h:1585
__INTRIN_INLINE unsigned int __readdr(unsigned int reg)
Definition: intrin_x86.h:1799
__INTRIN_INLINE char _InterlockedOr8(volatile char *value, char mask)
Definition: intrin_x86.h:486
__INTRIN_INLINE unsigned long __readcr0(void)
Definition: intrin_x86.h:1692
__INTRIN_INLINE unsigned long __readcr4(void)
Definition: intrin_x86.h:1713
GLsizei const GLfloat * value
Definition: glext.h:6069
__INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1437
__INTRIN_INLINE void _sgdt(void *Destination)
Definition: intrin_x86.h:1925
__INTRIN_INLINE unsigned long __indword(unsigned short Port)
Definition: intrin_x86.h:1419
__INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
Definition: intrin_x86.h:1372
#define local
Definition: zutil.h:30
__INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
Definition: intrin_x86.h:1895
__INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
Definition: intrin_x86.h:1329
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1530
__INTRIN_INLINE char _InterlockedAnd8(volatile char *value, char mask)
Definition: intrin_x86.h:429
#define byte(x, n)
Definition: tomcrypt.h:118
__INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:982
CPUINFO CPUInfo[]
Definition: parse.c:231
__INTRIN_INLINE void __incfsbyte(unsigned long Offset)
Definition: intrin_x86.h:950
__INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
Definition: intrin_x86.h:1510
__INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1243
static int reg
Definition: i386-dis.c:1275
void *__cdecl memmove(void *dest, const void *source, size_t num)
Definition: memmove.c:4
GLint GLint GLint GLint GLint GLint y
Definition: gl.h:1548
__INTRIN_INLINE short _InterlockedOr16(volatile short *value, short mask)
Definition: intrin_x86.h:505
const WCHAR * word
Definition: lex.c:78
__INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
Definition: intrin_x86.h:1504
__INTRIN_INLINE short _InterlockedExchange16(volatile short *Target, short Value)
Definition: intrin_x86.h:375
__INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short *Addend, short Value)
Definition: intrin_x86.h:411
__INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1131
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1865
__INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
Definition: intrin_x86.h:1219
__INTRIN_INLINE unsigned char _bittest(const long *a, long b)
Definition: intrin_x86.h:1012
__INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1193
__INTRIN_INLINE short _InterlockedXor16(volatile short *value, short mask)
Definition: intrin_x86.h:562
__INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
Definition: intrin_x86.h:1285
__INTRIN_INLINE void _mm_mfence(void)
Definition: intrin_x86.h:99
__INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
Definition: intrin_x86.h:1516
_In_ UINT _In_ UINT _In_ PNDIS_PACKET Source
Definition: ndis.h:3167
static char * dest
Definition: rtl.c:135
__INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1479
GLboolean GLboolean GLboolean GLboolean a
Definition: glext.h:6204
__INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
Definition: intrin_x86.h:1464
__INTRIN_INLINE unsigned short __inword(unsigned short Port)
Definition: intrin_x86.h:1412
__INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
Definition: intrin_x86.h:1459
__INTRIN_INLINE void __movsw(unsigned short *Destination, const unsigned short *Source, size_t Count)
Definition: intrin_x86.h:780
__INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
Definition: intrin_x86.h:1469
__INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long *a, long b)
Definition: intrin_x86.h:706
__INTRIN_INLINE unsigned char _bittestandcomplement(long *a, long b)
Definition: intrin_x86.h:1054
__INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
Definition: intrin_x86.h:1342