ReactOS  0.4.15-dev-1049-g1062a29
intrin_x86.h
Go to the documentation of this file.
1 /*
2  Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3  Microsoft Visual C++ functions. Originally developed for the ReactOS
4  (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5  projects.
6 
7  Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8 
9  Permission is hereby granted, free of charge, to any person obtaining a
10  copy of this software and associated documentation files (the "Software"),
11  to deal in the Software without restriction, including without limitation
12  the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  and/or sell copies of the Software, and to permit persons to whom the
14  Software is furnished to do so, subject to the following conditions:
15 
16  The above copyright notice and this permission notice shall be included in
17  all copies or substantial portions of the Software.
18 
19  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  DEALINGS IN THE SOFTWARE.
26 */
27 
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30 
31 /*
32  FIXME: review all "memory" clobbers, add/remove to match Visual C++
33  behavior: some "obvious" memory barriers are not present in the Visual C++
34  implementation - e.g. __stosX; on the other hand, some memory barriers that
35  *are* present could have been missed
36 */
37 
38 /*
39  NOTE: this is a *compatibility* header. Some functions may look wrong at
40  first, but they're only "as wrong" as they would be on Visual C++. Our
41  priority is compatibility
42 
43  NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44  constraints and the uses of __volatile__ out of my... hat. Do not touch
45  them. I hate cargo cult programming
46 
47  NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48  barriers aren't there in Visual C++ (e.g. __stosX)
49 
50  NOTE: review all intrinsics with a return value, add/remove __volatile__
51  where necessary. If an intrinsic whose value is ignored generates a no-op
52  under Visual C++, __volatile__ must be omitted; if it always generates code
53  (for example, if it has side effects), __volatile__ must be specified. GCC
54  will only optimize out non-volatile asm blocks with outputs, so input-only
55  blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56  and follow Visual C++ behavior
57 
58  NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59  atomic operations. Test the version like this:
60 
61  #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62  ...
63 
64  Pay attention to the type of barrier. Make it match with what Visual C++
65  would use in the same case
66 */
67 
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71 
72 /*** memcopy must be memmove ***/
73 void* __cdecl memmove(void* dest, const void* source, size_t num);
74 __INTRIN_INLINE void* __cdecl memcpy(void* dest, const void* source, size_t num)
75 {
76  return memmove(dest, source, num);
77 }
78 
79 
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
84 
85 /*** Memory barriers ***/
86 
87 #if !HAS_BUILTIN(_ReadWriteBarrier)
89 {
90  __asm__ __volatile__("" : : : "memory");
91 }
92 #endif
93 
94 /* GCC only supports full barriers */
95 #define _ReadBarrier _ReadWriteBarrier
96 #define _WriteBarrier _ReadWriteBarrier
97 
98 #if !HAS_BUILTIN(_mm_mfence)
100 {
101  __asm__ __volatile__("mfence" : : : "memory");
102 }
103 #endif
104 
105 #if !HAS_BUILTIN(_mm_lfence)
107 {
108  _ReadBarrier();
109  __asm__ __volatile__("lfence");
110  _ReadBarrier();
111 }
112 #endif
113 
114 #if !HAS_BUILTIN(_mm_sfence)
116 {
117  _WriteBarrier();
118  __asm__ __volatile__("sfence");
119  _WriteBarrier();
120 }
121 #endif
122 
123 #ifdef __x86_64__
124 __INTRIN_INLINE void __faststorefence(void)
125 {
126  long local;
127  __asm__ __volatile__("lock; orl $0, %0;" : : "m"(local));
128 }
129 #endif
130 
131 
132 /*** Atomic operations ***/
133 
134 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
135 
136 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
137 {
138  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
139 }
140 
141 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
142 {
143  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
144 }
145 
146 #if !HAS_BUILTIN(_InterlockedCompareExchange)
147 __INTRIN_INLINE long __cdecl _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
148 {
149  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
150 }
151 #endif
152 
153 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
154 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
155 {
156  return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
157 }
158 #endif
159 
160 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
161 {
162  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
163  __sync_synchronize();
164  return __sync_lock_test_and_set(Target, Value);
165 }
166 
167 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
168 {
169  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
170  __sync_synchronize();
171  return __sync_lock_test_and_set(Target, Value);
172 }
173 
174 #if !HAS_BUILTIN(_InterlockedExchange)
175 __INTRIN_INLINE long __cdecl _InterlockedExchange(volatile long * Target, long Value)
176 {
177  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
178  __sync_synchronize();
179  return __sync_lock_test_and_set(Target, Value);
180 }
181 #endif
182 
183 #if !HAS_BUILTIN(_InterlockedExchangePointer)
184 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
185 {
186  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
187  __sync_synchronize();
188  return (void *)__sync_lock_test_and_set(Target, Value);
189 }
190 #endif
191 
192 #if defined(__x86_64__)
193 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * Target, long long Value)
194 {
195  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
196  __sync_synchronize();
197  return __sync_lock_test_and_set(Target, Value);
198 }
199 #endif
200 
201 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
202 {
203  return __sync_fetch_and_add(Addend, Value);
204 }
205 
206 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
207 {
208  return __sync_fetch_and_add(Addend, Value);
209 }
210 
211 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
212 __INTRIN_INLINE long __cdecl _InterlockedExchangeAdd(volatile long * Addend, long Value)
213 {
214  return __sync_fetch_and_add(Addend, Value);
215 }
216 #endif
217 
218 #if defined(__x86_64__)
219 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * Addend, long long Value)
220 {
221  return __sync_fetch_and_add(Addend, Value);
222 }
223 #endif
224 
225 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
226 {
227  return __sync_fetch_and_and(value, mask);
228 }
229 
230 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
231 {
232  return __sync_fetch_and_and(value, mask);
233 }
234 
235 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
236 {
237  return __sync_fetch_and_and(value, mask);
238 }
239 
240 #if defined(__x86_64__)
241 __INTRIN_INLINE long long _InterlockedAnd64(volatile long long * value, long long mask)
242 {
243  return __sync_fetch_and_and(value, mask);
244 }
245 #endif
246 
247 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
248 {
249  return __sync_fetch_and_or(value, mask);
250 }
251 
252 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
253 {
254  return __sync_fetch_and_or(value, mask);
255 }
256 
257 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
258 {
259  return __sync_fetch_and_or(value, mask);
260 }
261 
262 #if defined(__x86_64__)
263 __INTRIN_INLINE long long _InterlockedOr64(volatile long long * value, long long mask)
264 {
265  return __sync_fetch_and_or(value, mask);
266 }
267 #endif
268 
269 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
270 {
271  return __sync_fetch_and_xor(value, mask);
272 }
273 
274 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
275 {
276  return __sync_fetch_and_xor(value, mask);
277 }
278 
279 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
280 {
281  return __sync_fetch_and_xor(value, mask);
282 }
283 
284 #if defined(__x86_64__)
285 __INTRIN_INLINE long long _InterlockedXor64(volatile long long * value, long long mask)
286 {
287  return __sync_fetch_and_xor(value, mask);
288 }
289 #endif
290 
291 #if !HAS_BUILTIN(_InterlockedDecrement)
292 __INTRIN_INLINE long __cdecl _InterlockedDecrement(volatile long * lpAddend)
293 {
294  return __sync_sub_and_fetch(lpAddend, 1);
295 }
296 #endif
297 
298 #if !HAS_BUILTIN(_InterlockedIncrement)
299 __INTRIN_INLINE long __cdecl _InterlockedIncrement(volatile long * lpAddend)
300 {
301  return __sync_add_and_fetch(lpAddend, 1);
302 }
303 #endif
304 
305 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
306 {
307  return __sync_sub_and_fetch(lpAddend, 1);
308 }
309 
310 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
311 {
312  return __sync_add_and_fetch(lpAddend, 1);
313 }
314 
315 #if defined(__x86_64__)
316 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
317 {
318  return __sync_sub_and_fetch(lpAddend, 1);
319 }
320 
321 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
322 {
323  return __sync_add_and_fetch(lpAddend, 1);
324 }
325 #endif
326 
327 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
328 
329 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
330 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
331 {
332  char retval = Comperand;
333  __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
334  return retval;
335 }
336 #endif
337 
338 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
339 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
340 {
341  short retval = Comperand;
342  __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
343  return retval;
344 }
345 #endif
346 
347 #if !HAS_BUILTIN(_InterlockedCompareExchange)
348 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
349 {
350  long retval = Comperand;
351  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
352  return retval;
353 }
354 #endif
355 
356 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
357 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
358 {
359  void * retval = (void *)Comperand;
360  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
361  return retval;
362 }
363 #endif
364 
365 #if !HAS_BUILTIN(_InterlockedExchange8)
367 {
368  char retval = Value;
369  __asm__("xchgb %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
370  return retval;
371 }
372 #endif
373 
374 #if !HAS_BUILTIN(_InterlockedExchange16)
375 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
376 {
377  short retval = Value;
378  __asm__("xchgw %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
379  return retval;
380 }
381 #endif
382 
383 #if !HAS_BUILTIN(_InterlockedExchange)
385 {
386  long retval = Value;
387  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
388  return retval;
389 }
390 #endif
391 
392 #if !HAS_BUILTIN(_InterlockedExchangePointer)
393 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
394 {
395  void * retval = Value;
396  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
397  return retval;
398 }
399 #endif
400 
401 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
403 {
404  char retval = Value;
405  __asm__("lock; xaddb %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
406  return retval;
407 }
408 #endif
409 
410 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
412 {
413  short retval = Value;
414  __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
415  return retval;
416 }
417 #endif
418 
419 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
421 {
422  long retval = Value;
423  __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
424  return retval;
425 }
426 #endif
427 
428 #if !HAS_BUILTIN(_InterlockedAnd8)
429 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
430 {
431  char x;
432  char y;
433 
434  y = *value;
435 
436  do
437  {
438  x = y;
440  }
441  while(y != x);
442 
443  return y;
444 }
445 #endif
446 
447 #if !HAS_BUILTIN(_InterlockedAnd16)
448 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
449 {
450  short x;
451  short y;
452 
453  y = *value;
454 
455  do
456  {
457  x = y;
459  }
460  while(y != x);
461 
462  return y;
463 }
464 #endif
465 
466 #if !HAS_BUILTIN(_InterlockedAnd)
467 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
468 {
469  long x;
470  long y;
471 
472  y = *value;
473 
474  do
475  {
476  x = y;
478  }
479  while(y != x);
480 
481  return y;
482 }
483 #endif
484 
485 #if !HAS_BUILTIN(_InterlockedOr8)
486 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
487 {
488  char x;
489  char y;
490 
491  y = *value;
492 
493  do
494  {
495  x = y;
497  }
498  while(y != x);
499 
500  return y;
501 }
502 #endif
503 
504 #if !HAS_BUILTIN(_InterlockedOr16)
505 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
506 {
507  short x;
508  short y;
509 
510  y = *value;
511 
512  do
513  {
514  x = y;
516  }
517  while(y != x);
518 
519  return y;
520 }
521 #endif
522 
523 #if !HAS_BUILTIN(_InterlockedOr)
524 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
525 {
526  long x;
527  long y;
528 
529  y = *value;
530 
531  do
532  {
533  x = y;
535  }
536  while(y != x);
537 
538  return y;
539 }
540 #endif
541 
542 #if !HAS_BUILTIN(_InterlockedXor8)
543 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
544 {
545  char x;
546  char y;
547 
548  y = *value;
549 
550  do
551  {
552  x = y;
554  }
555  while(y != x);
556 
557  return y;
558 }
559 #endif
560 
561 #if !HAS_BUILTIN(_InterlockedXor16)
562 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
563 {
564  short x;
565  short y;
566 
567  y = *value;
568 
569  do
570  {
571  x = y;
573  }
574  while(y != x);
575 
576  return y;
577 }
578 #endif
579 
580 #if !HAS_BUILTIN(_InterlockedXor)
581 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
582 {
583  long x;
584  long y;
585 
586  y = *value;
587 
588  do
589  {
590  x = y;
592  }
593  while(y != x);
594 
595  return y;
596 }
597 #endif
598 
599 #if !HAS_BUILTIN(_InterlockedDecrement)
600 __INTRIN_INLINE long _InterlockedDecrement(volatile long * lpAddend)
601 {
602  return _InterlockedExchangeAdd(lpAddend, -1) - 1;
603 }
604 #endif
605 
606 #if !HAS_BUILTIN(_InterlockedIncrement)
607 __INTRIN_INLINE long _InterlockedIncrement(volatile long * lpAddend)
608 {
609  return _InterlockedExchangeAdd(lpAddend, 1) + 1;
610 }
611 #endif
612 
613 #if !HAS_BUILTIN(_InterlockedDecrement16)
614 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
615 {
616  return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
617 }
618 #endif
619 
620 #if !HAS_BUILTIN(_InterlockedIncrement16)
621 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
622 {
623  return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
624 }
625 #endif
626 
627 #if defined(__x86_64__)
628 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
629 {
630  return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
631 }
632 
633 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
634 {
635  return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
636 }
637 #endif
638 
639 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
640 
641 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
642 
643 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
644 {
645  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
646 }
647 
648 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
649 
650 #if !HAS_BUILTIN(_InterlockedCompareExchange64)
651 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
652 {
653  long long retval = Comperand;
654 
655  __asm__
656  (
657  "lock; cmpxchg8b %[Destination]" :
658  [retval] "+A" (retval) :
659  [Destination] "m" (*Destination),
660  "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
661  "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
662  "memory"
663  );
664 
665  return retval;
666 }
667 #endif
668 
669 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
670 
671 #ifdef __i386__
672 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * Addend, long Value)
673 {
674  __asm__
675  (
676  "lock; addl %[Value], %[Lo32];"
677  "jae LABEL%=;"
678  "lock; adcl $0, %[Hi32];"
679  "LABEL%=:;" :
680  [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
681  [Value] "ir" (Value) :
682  "memory"
683  );
684 
685  return Value;
686 }
687 #endif /* __i386__ */
688 
689 #if !HAS_BUILTIN(_interlockedbittestandreset)
690 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, long b)
691 {
692  unsigned char retval;
693  __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
694  return retval;
695 }
696 #endif
697 
698 #if defined(__x86_64__)
699 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, long long b)
700 {
701  unsigned char retval;
702  __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
703  return retval;
704 }
705 #endif
706 
707 #if !HAS_BUILTIN(_interlockedbittestandset)
708 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, long b)
709 {
710  unsigned char retval;
711  __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
712  return retval;
713 }
714 #endif
715 
716 #if defined(__x86_64__)
717 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, long long b)
718 {
719  unsigned char retval;
720  __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
721  return retval;
722 }
723 #endif
724 
725 /*** String operations ***/
726 
727 #if !HAS_BUILTIN(__stosb)
728 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
729 __INTRIN_INLINE void __stosb(unsigned char * Dest, unsigned char Data, size_t Count)
730 {
731  __asm__ __volatile__
732  (
733  "rep; stosb" :
734  [Dest] "=D" (Dest), [Count] "=c" (Count) :
735  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
736  );
737 }
738 #endif
739 
740 __INTRIN_INLINE void __stosw(unsigned short * Dest, unsigned short Data, size_t Count)
741 {
742  __asm__ __volatile__
743  (
744  "rep; stosw" :
745  [Dest] "=D" (Dest), [Count] "=c" (Count) :
746  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
747  );
748 }
749 
750 __INTRIN_INLINE void __stosd(unsigned long * Dest, unsigned long Data, size_t Count)
751 {
752  __asm__ __volatile__
753  (
754  "rep; stosl" :
755  [Dest] "=D" (Dest), [Count] "=c" (Count) :
756  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
757  );
758 }
759 
760 #ifdef __x86_64__
761 __INTRIN_INLINE void __stosq(unsigned long long * Dest, unsigned long long Data, size_t Count)
762 {
763  __asm__ __volatile__
764  (
765  "rep; stosq" :
766  [Dest] "=D" (Dest), [Count] "=c" (Count) :
767  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
768  );
769 }
770 #endif
771 
772 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
773 {
774  __asm__ __volatile__
775  (
776  "rep; movsb" :
777  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
778  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
779  );
780 }
781 
782 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
783 {
784  __asm__ __volatile__
785  (
786  "rep; movsw" :
787  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
788  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
789  );
790 }
791 
792 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
793 {
794  __asm__ __volatile__
795  (
796  "rep; movsd" :
797  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
798  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
799  );
800 }
801 
802 #ifdef __x86_64__
803 __INTRIN_INLINE void __movsq(unsigned long * Destination, const unsigned long * Source, size_t Count)
804 {
805  __asm__ __volatile__
806  (
807  "rep; movsq" :
808  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
809  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
810  );
811 }
812 #endif
813 
814 #if defined(__x86_64__)
815 
816 /*** GS segment addressing ***/
817 
818 __INTRIN_INLINE void __writegsbyte(unsigned long Offset, unsigned char Data)
819 {
820  __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
821 }
822 
823 __INTRIN_INLINE void __writegsword(unsigned long Offset, unsigned short Data)
824 {
825  __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
826 }
827 
828 __INTRIN_INLINE void __writegsdword(unsigned long Offset, unsigned long Data)
829 {
830  __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
831 }
832 
833 __INTRIN_INLINE void __writegsqword(unsigned long Offset, unsigned long long Data)
834 {
835  __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
836 }
837 
838 __INTRIN_INLINE unsigned char __readgsbyte(unsigned long Offset)
839 {
840  unsigned char value;
841  __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
842  return value;
843 }
844 
845 __INTRIN_INLINE unsigned short __readgsword(unsigned long Offset)
846 {
847  unsigned short value;
848  __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
849  return value;
850 }
851 
852 __INTRIN_INLINE unsigned long __readgsdword(unsigned long Offset)
853 {
854  unsigned long value;
855  __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
856  return value;
857 }
858 
859 __INTRIN_INLINE unsigned long long __readgsqword(unsigned long Offset)
860 {
861  unsigned long long value;
862  __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
863  return value;
864 }
865 
866 __INTRIN_INLINE void __incgsbyte(unsigned long Offset)
867 {
868  __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
869 }
870 
871 __INTRIN_INLINE void __incgsword(unsigned long Offset)
872 {
873  __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
874 }
875 
876 __INTRIN_INLINE void __incgsdword(unsigned long Offset)
877 {
878  __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
879 }
880 
881 __INTRIN_INLINE void __incgsqword(unsigned long Offset)
882 {
883  __asm__ __volatile__("incq %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
884 }
885 
886 __INTRIN_INLINE void __addgsbyte(unsigned long Offset, unsigned char Data)
887 {
888  __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
889 }
890 
891 __INTRIN_INLINE void __addgsword(unsigned long Offset, unsigned short Data)
892 {
893  __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
894 }
895 
896 __INTRIN_INLINE void __addgsdword(unsigned long Offset, unsigned int Data)
897 {
898  __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
899 }
900 
901 __INTRIN_INLINE void __addgsqword(unsigned long Offset, unsigned long long Data)
902 {
903  __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
904 }
905 
906 #else /* defined(__x86_64__) */
907 
908 /*** FS segment addressing ***/
909 
910 __INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
911 {
912  __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
913 }
914 
915 __INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
916 {
917  __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
918 }
919 
920 __INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
921 {
922  __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
923 }
924 
925 #if !HAS_BUILTIN(__readfsbyte)
926 __INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
927 {
928  unsigned char value;
929  __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
930  return value;
931 }
932 #endif
933 
934 #if !HAS_BUILTIN(__readfsword)
935 __INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
936 {
937  unsigned short value;
938  __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
939  return value;
940 }
941 #endif
942 
943 #if !HAS_BUILTIN(__readfsdword)
944 __INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
945 {
946  unsigned long value;
947  __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
948  return value;
949 }
950 #endif
951 
952 __INTRIN_INLINE void __incfsbyte(unsigned long Offset)
953 {
954  __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
955 }
956 
957 __INTRIN_INLINE void __incfsword(unsigned long Offset)
958 {
959  __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
960 }
961 
963 {
964  __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
965 }
966 
967 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
968 __INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
969 {
970  if(!__builtin_constant_p(Offset))
971  __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
972  else
973  __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
974 }
975 
976 __INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
977 {
978  if(!__builtin_constant_p(Offset))
979  __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
980  else
981  __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
982 }
983 
984 __INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
985 {
986  if(!__builtin_constant_p(Offset))
987  __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
988  else
989  __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
990 }
991 
992 #endif /* defined(__x86_64__) */
993 
994 
995 /*** Bit manipulation ***/
996 
997 #if !HAS_BUILTIN(_BitScanForward)
998 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * Index, unsigned long Mask)
999 {
1000  __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1001  return Mask ? 1 : 0;
1002 }
1003 #endif
1004 
1005 #if !HAS_BUILTIN(_BitScanReverse)
1006 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * Index, unsigned long Mask)
1007 {
1008  __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1009  return Mask ? 1 : 0;
1010 }
1011 #endif
1012 
1013 #if !HAS_BUILTIN(_bittest)
1014 /* NOTE: again, the bizarre implementation follows Visual C++ */
1015 __INTRIN_INLINE unsigned char _bittest(const long * a, long b)
1016 {
1017  unsigned char retval;
1018 
1019  if(__builtin_constant_p(b))
1020  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
1021  else
1022  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1023 
1024  return retval;
1025 }
1026 #endif
1027 
1028 #ifdef __x86_64__
1029 __INTRIN_INLINE unsigned char _BitScanForward64(unsigned long * Index, unsigned long long Mask)
1030 {
1031  unsigned long long Index64;
1032  __asm__("bsfq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1033  *Index = Index64;
1034  return Mask ? 1 : 0;
1035 }
1036 
1037 __INTRIN_INLINE unsigned char _BitScanReverse64(unsigned long * Index, unsigned long long Mask)
1038 {
1039  unsigned long long Index64;
1040  __asm__("bsrq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1041  *Index = Index64;
1042  return Mask ? 1 : 0;
1043 }
1044 
1045 __INTRIN_INLINE unsigned char _bittest64(const long long * a, long long b)
1046 {
1047  unsigned char retval;
1048 
1049  if(__builtin_constant_p(b))
1050  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
1051  else
1052  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1053 
1054  return retval;
1055 }
1056 #endif
1057 
1058 #if !HAS_BUILTIN(_bittestandcomplement)
1059 __INTRIN_INLINE unsigned char _bittestandcomplement(long * a, long b)
1060 {
1061  unsigned char retval;
1062 
1063  if(__builtin_constant_p(b))
1064  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1065  else
1066  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1067 
1068  return retval;
1069 }
1070 #endif
1071 
1072 #if !HAS_BUILTIN(_bittestandreset)
1073 __INTRIN_INLINE unsigned char _bittestandreset(long * a, long b)
1074 {
1075  unsigned char retval;
1076 
1077  if(__builtin_constant_p(b))
1078  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1079  else
1080  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1081 
1082  return retval;
1083 }
1084 #endif
1085 
1086 #if !HAS_BUILTIN(_bittestandset)
1087 __INTRIN_INLINE unsigned char _bittestandset(long * a, long b)
1088 {
1089  unsigned char retval;
1090 
1091  if(__builtin_constant_p(b))
1092  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1093  else
1094  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1095 
1096  return retval;
1097 }
1098 #endif
1099 
1100 #ifdef __x86_64__
1101 
1102 __INTRIN_INLINE unsigned char _bittestandset64(long long * a, long long b)
1103 {
1104  unsigned char retval;
1105 
1106  if(__builtin_constant_p(b))
1107  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1108  else
1109  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1110 
1111  return retval;
1112 }
1113 
1114 __INTRIN_INLINE unsigned char _bittestandreset64(long long * a, long long b)
1115 {
1116  unsigned char retval;
1117 
1118  if(__builtin_constant_p(b))
1119  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1120  else
1121  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1122 
1123  return retval;
1124 }
1125 
1126 __INTRIN_INLINE unsigned char _bittestandcomplement64(long long * a, long long b)
1127 {
1128  unsigned char retval;
1129 
1130  if(__builtin_constant_p(b))
1131  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1132  else
1133  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1134 
1135  return retval;
1136 }
1137 
1138 #endif /* __x86_64__ */
1139 
1140 #if !HAS_BUILTIN(_rotl8)
1141 __INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
1142 {
1143  unsigned char retval;
1144  __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1145  return retval;
1146 }
1147 #endif
1148 
1149 #if !HAS_BUILTIN(_rotl16)
1150 __INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
1151 {
1152  unsigned short retval;
1153  __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1154  return retval;
1155 }
1156 #endif
1157 
1158 #if !HAS_BUILTIN(_rotl)
1159 __INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
1160 {
1161  unsigned int retval;
1162  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1163  return retval;
1164 }
1165 #endif
1166 
1167 #ifdef __x86_64__
1168 __INTRIN_INLINE unsigned long long _rotl64(unsigned long long value, int shift)
1169 {
1170  unsigned long long retval;
1171  __asm__("rolq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1172  return retval;
1173 }
1174 #else /* __x86_64__ */
1175 #if !HAS_BUILTIN(_rotl64)
1176 __INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
1177 {
1178  /* FIXME: this is probably not optimal */
1179  return (value << shift) | (value >> (64 - shift));
1180 }
1181 #endif /* !HAS_BUILTIN(_rotl64) */
1182 #endif /* __x86_64__ */
1183 
1184 #if !HAS_BUILTIN(_rotr)
1185 __INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
1186 {
1187  unsigned int retval;
1188  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1189  return retval;
1190 }
1191 #endif
1192 
1193 #if !HAS_BUILTIN(_rotr8)
1194 __INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
1195 {
1196  unsigned char retval;
1197  __asm__("rorb %b[shift], %b[retval]" : [retval] "=qm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1198  return retval;
1199 }
1200 #endif
1201 
1202 #if !HAS_BUILTIN(_rotr16)
1203 __INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
1204 {
1205  unsigned short retval;
1206  __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1207  return retval;
1208 }
1209 #endif
1210 
1211 #ifdef __x86_64__
1212 __INTRIN_INLINE unsigned long long _rotr64(unsigned long long value, int shift)
1213 {
1214  unsigned long long retval;
1215  __asm__("rorq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1216  return retval;
1217 }
1218 #else /* __x86_64__ */
1219 #if !HAS_BUILTIN(_rotr64)
1220 __INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
1221 {
1222  /* FIXME: this is probably not optimal */
1223  return (value >> shift) | (value << (64 - shift));
1224 }
1225 #endif /* !HAS_BUILTIN(_rotr64) */
1226 #endif /* __x86_64__ */
1227 
1228 #if !HAS_BUILTIN(_lrotl)
1229 __INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
1230 {
1231  unsigned long retval;
1232  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1233  return retval;
1234 }
1235 #endif
1236 
1237 #if !HAS_BUILTIN(_lrotr)
1238 __INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
1239 {
1240  unsigned long retval;
1241  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1242  return retval;
1243 }
1244 #endif
1245 
1246 /*
1247  NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1248  constraint (edx:eax) for the Mask argument, because it's the only way GCC
1249  can pass 64-bit operands around - passing the two 32 bit parts separately
1250  just confuses it. Also we declare Bit as an int and then truncate it to
1251  match Visual C++ behavior
1252 */
1253 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1254 {
1255  unsigned long long retval = Mask;
1256 
1257  __asm__
1258  (
1259  "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1260  "+A" (retval) :
1261  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1262  );
1263 
1264  return retval;
1265 }
1266 
1267 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1268 {
1269  long long retval = Mask;
1270 
1271  __asm__
1272  (
1273  "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1274  "+A" (retval) :
1275  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1276  );
1277 
1278  return retval;
1279 }
1280 
1281 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1282 {
1283  unsigned long long retval = Mask;
1284 
1285  __asm__
1286  (
1287  "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1288  "+A" (retval) :
1289  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1290  );
1291 
1292  return retval;
1293 }
1294 
1295 __INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
1296 {
1297  unsigned short retval;
1298  __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
1299  return retval;
1300 }
1301 
1302 __INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
1303 {
1304  unsigned long retval;
1305  __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1306  return retval;
1307 }
1308 
1309 #ifdef __x86_64__
1310 __INTRIN_INLINE unsigned long long _byteswap_uint64(unsigned long long value)
1311 {
1312  unsigned long long retval;
1313  __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1314  return retval;
1315 }
1316 #else
1317 __INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
1318 {
1319  union {
1320  unsigned long long int64part;
1321  struct {
1322  unsigned long lowpart;
1323  unsigned long hipart;
1324  };
1325  } retval;
1326  retval.int64part = value;
1327  __asm__("bswapl %[lowpart]\n"
1328  "bswapl %[hipart]\n"
1329  : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
1330  return retval.int64part;
1331 }
1332 #endif
1333 
1334 #if !HAS_BUILTIN(__lzcnt)
1335 __INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
1336 {
1337  return __builtin_clz(value);
1338 }
1339 #endif
1340 
1341 #if !HAS_BUILTIN(__lzcnt16)
1342 __INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
1343 {
1344  return __builtin_clz(value);
1345 }
1346 #endif
1347 
1348 #if !HAS_BUILTIN(__popcnt)
1349 __INTRIN_INLINE unsigned int __popcnt(unsigned int value)
1350 {
1351  return __builtin_popcount(value);
1352 }
1353 #endif
1354 
1355 #if !HAS_BUILTIN(__popcnt16)
1356 __INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
1357 {
1358  return __builtin_popcount(value);
1359 }
1360 #endif
1361 
1362 #ifdef __x86_64__
1363 __INTRIN_INLINE unsigned long long __lzcnt64(unsigned long long value)
1364 {
1365  return __builtin_clzll(value);
1366 }
1367 
1368 __INTRIN_INLINE unsigned long long __popcnt64(unsigned long long value)
1369 {
1370  return __builtin_popcountll(value);
1371 }
1372 #endif
1373 
1374 /*** 64-bit math ***/
1375 
1376 #if !HAS_BUILTIN(__emul)
1377 __INTRIN_INLINE long long __emul(int a, int b)
1378 {
1379  long long retval;
1380  __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1381  return retval;
1382 }
1383 #endif
1384 
1385 #if !HAS_BUILTIN(__emulu)
1386 __INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
1387 {
1388  unsigned long long retval;
1389  __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1390  return retval;
1391 }
1392 #endif
1393 
1394 __INTRIN_INLINE long long __cdecl _abs64(long long value)
1395 {
1396  return (value >= 0) ? value : -value;
1397 }
1398 
1399 #ifdef __x86_64__
1400 
1401 __INTRIN_INLINE long long __mulh(long long a, long long b)
1402 {
1403  long long retval;
1404  __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1405  return retval;
1406 }
1407 
1408 __INTRIN_INLINE unsigned long long __umulh(unsigned long long a, unsigned long long b)
1409 {
1410  unsigned long long retval;
1411  __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1412  return retval;
1413 }
1414 
1415 #endif
1416 
1417 /*** Port I/O ***/
1418 
1419 __INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
1420 {
1421  unsigned char byte;
1422  __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1423  return byte;
1424 }
1425 
1426 __INTRIN_INLINE unsigned short __inword(unsigned short Port)
1427 {
1428  unsigned short word;
1429  __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1430  return word;
1431 }
1432 
1433 __INTRIN_INLINE unsigned long __indword(unsigned short Port)
1434 {
1435  unsigned long dword;
1436  __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1437  return dword;
1438 }
1439 
1440 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1441 {
1442  __asm__ __volatile__
1443  (
1444  "rep; insb" :
1445  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1446  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1447  "memory"
1448  );
1449 }
1450 
1451 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1452 {
1453  __asm__ __volatile__
1454  (
1455  "rep; insw" :
1456  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1457  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1458  "memory"
1459  );
1460 }
1461 
1462 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1463 {
1464  __asm__ __volatile__
1465  (
1466  "rep; insl" :
1467  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1468  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1469  "memory"
1470  );
1471 }
1472 
1473 __INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
1474 {
1475  __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1476 }
1477 
1478 __INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
1479 {
1480  __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1481 }
1482 
1483 __INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
1484 {
1485  __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1486 }
1487 
1488 __INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1489 {
1490  __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1491 }
1492 
1493 __INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1494 {
1495  __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1496 }
1497 
1498 __INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1499 {
1500  __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1501 }
1502 
1503 __INTRIN_INLINE int __cdecl _inp(unsigned short Port)
1504 {
1505  return __inbyte(Port);
1506 }
1507 
1508 __INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
1509 {
1510  return __inword(Port);
1511 }
1512 
1513 __INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
1514 {
1515  return __indword(Port);
1516 }
1517 
1518 __INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
1519 {
1520  __outbyte(Port, (unsigned char)databyte);
1521  return databyte;
1522 }
1523 
1524 __INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
1525 {
1526  __outword(Port, dataword);
1527  return dataword;
1528 }
1529 
1530 __INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
1531 {
1532  __outdword(Port, dataword);
1533  return dataword;
1534 }
1535 
1536 
1537 /*** System information ***/
1538 
1539 __INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
1540 {
1541  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1542 }
1543 
1544 __INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
1545 {
1546  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType), "c" (ECXValue));
1547 }
1548 
1549 #if !HAS_BUILTIN(__rdtsc)
1550 __INTRIN_INLINE unsigned long long __rdtsc(void)
1551 {
1552 #ifdef __x86_64__
1553  unsigned long long low, high;
1554  __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1555  return low | (high << 32);
1556 #else
1557  unsigned long long retval;
1558  __asm__ __volatile__("rdtsc" : "=A"(retval));
1559  return retval;
1560 #endif
1561 }
1562 #endif /* !HAS_BUILTIN(__rdtsc) */
1563 
1565 {
1566  __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1567 }
1568 
1570 {
1571  uintptr_t retval;
1572  __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1573  return retval;
1574 }
1575 
1576 /*** Interrupts ***/
1577 
1578 #if !HAS_BUILTIN(__debugbreak)
1580 {
1581  __asm__("int $3");
1582 }
1583 #endif
1584 
1585 #if !HAS_BUILTIN(__ud2)
1587 {
1588  __asm__("ud2");
1589 }
1590 #endif
1591 
1592 #if !HAS_BUILTIN(__int2c)
1594 {
1595  __asm__("int $0x2c");
1596 }
1597 #endif
1598 
1600 {
1601  __asm__("cli" : : : "memory");
1602 }
1603 
1605 {
1606  __asm__("sti" : : : "memory");
1607 }
1608 
1610 {
1611  __asm__("hlt" : : : "memory");
1612 }
1613 
1614 #if !HAS_BUILTIN(__fastfail)
1615 __declspec(noreturn)
1616 __INTRIN_INLINE void __fastfail(unsigned int Code)
1617 {
1618  __asm__("int $0x29" : : "c"(Code) : "memory");
1619  __builtin_unreachable();
1620 }
1621 #endif
1622 
1623 /*** Protected memory management ***/
1624 
1625 #ifdef __x86_64__
1626 
1627 __INTRIN_INLINE void __writecr0(unsigned long long Data)
1628 {
1629  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1630 }
1631 
1632 __INTRIN_INLINE void __writecr3(unsigned long long Data)
1633 {
1634  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1635 }
1636 
1637 __INTRIN_INLINE void __writecr4(unsigned long long Data)
1638 {
1639  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1640 }
1641 
1642 __INTRIN_INLINE void __writecr8(unsigned long long Data)
1643 {
1644  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1645 }
1646 
1647 __INTRIN_INLINE unsigned long long __readcr0(void)
1648 {
1649  unsigned long long value;
1650  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1651  return value;
1652 }
1653 
1654 __INTRIN_INLINE unsigned long long __readcr2(void)
1655 {
1656  unsigned long long value;
1657  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1658  return value;
1659 }
1660 
1661 __INTRIN_INLINE unsigned long long __readcr3(void)
1662 {
1663  unsigned long long value;
1664  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1665  return value;
1666 }
1667 
1668 __INTRIN_INLINE unsigned long long __readcr4(void)
1669 {
1670  unsigned long long value;
1671  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1672  return value;
1673 }
1674 
1675 __INTRIN_INLINE unsigned long long __readcr8(void)
1676 {
1677  unsigned long long value;
1678  __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1679  return value;
1680 }
1681 
1682 #else /* __x86_64__ */
1683 
1684 __INTRIN_INLINE void __writecr0(unsigned int Data)
1685 {
1686  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1687 }
1688 
1689 __INTRIN_INLINE void __writecr3(unsigned int Data)
1690 {
1691  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1692 }
1693 
1694 __INTRIN_INLINE void __writecr4(unsigned int Data)
1695 {
1696  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1697 }
1698 
1699 #if !HAS_BUILTIN(__writecr8)
1700 __INTRIN_INLINE void __writecr8(unsigned int Data)
1701 {
1702  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1703 }
1704 #endif
1705 
1706 __INTRIN_INLINE unsigned long __readcr0(void)
1707 {
1708  unsigned long value;
1709  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1710  return value;
1711 }
1712 
1713 __INTRIN_INLINE unsigned long __readcr2(void)
1714 {
1715  unsigned long value;
1716  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1717  return value;
1718 }
1719 
1720 __INTRIN_INLINE unsigned long __readcr3(void)
1721 {
1722  unsigned long value;
1723  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1724  return value;
1725 }
1726 
1727 __INTRIN_INLINE unsigned long __readcr4(void)
1728 {
1729  unsigned long value;
1730  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1731  return value;
1732 }
1733 
1734 #if !HAS_BUILTIN(__readcr8)
1735 __INTRIN_INLINE unsigned long __readcr8(void)
1736 {
1737  unsigned long value;
1738  __asm__ __volatile__("mov %%cr8, %[value]" : [value] "=r" (value));
1739  return value;
1740 }
1741 #endif
1742 
1743 #endif /* __x86_64__ */
1744 
1745 #ifdef __x86_64__
1746 
1747 __INTRIN_INLINE unsigned long long __readdr(unsigned int reg)
1748 {
1749  unsigned long long value;
1750  switch (reg)
1751  {
1752  case 0:
1753  __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1754  break;
1755  case 1:
1756  __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1757  break;
1758  case 2:
1759  __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1760  break;
1761  case 3:
1762  __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1763  break;
1764  case 4:
1765  __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1766  break;
1767  case 5:
1768  __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1769  break;
1770  case 6:
1771  __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1772  break;
1773  case 7:
1774  __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1775  break;
1776  }
1777  return value;
1778 }
1779 
1780 __INTRIN_INLINE void __writedr(unsigned reg, unsigned long long value)
1781 {
1782  switch (reg)
1783  {
1784  case 0:
1785  __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1786  break;
1787  case 1:
1788  __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1789  break;
1790  case 2:
1791  __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1792  break;
1793  case 3:
1794  __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1795  break;
1796  case 4:
1797  __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1798  break;
1799  case 5:
1800  __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1801  break;
1802  case 6:
1803  __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1804  break;
1805  case 7:
1806  __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1807  break;
1808  }
1809 }
1810 
1811 #else /* __x86_64__ */
1812 
1813 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1814 {
1815  unsigned int value;
1816  switch (reg)
1817  {
1818  case 0:
1819  __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1820  break;
1821  case 1:
1822  __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1823  break;
1824  case 2:
1825  __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1826  break;
1827  case 3:
1828  __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1829  break;
1830  case 4:
1831  __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1832  break;
1833  case 5:
1834  __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1835  break;
1836  case 6:
1837  __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1838  break;
1839  case 7:
1840  __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1841  break;
1842  }
1843  return value;
1844 }
1845 
1846 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1847 {
1848  switch (reg)
1849  {
1850  case 0:
1851  __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1852  break;
1853  case 1:
1854  __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1855  break;
1856  case 2:
1857  __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1858  break;
1859  case 3:
1860  __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1861  break;
1862  case 4:
1863  __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1864  break;
1865  case 5:
1866  __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1867  break;
1868  case 6:
1869  __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1870  break;
1871  case 7:
1872  __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1873  break;
1874  }
1875 }
1876 
1877 #endif /* __x86_64__ */
1878 
1880 {
1881  __asm__ __volatile__ ("invlpg (%[Address])" : : [Address] "b" (Address) : "memory");
1882 }
1883 
1884 
1885 /*** System operations ***/
1886 
1887 __INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
1888 {
1889 #ifdef __x86_64__
1890  unsigned long low, high;
1891  __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1892  return ((unsigned long long)high << 32) | low;
1893 #else
1894  unsigned long long retval;
1895  __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1896  return retval;
1897 #endif
1898 }
1899 
1900 __INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
1901 {
1902 #ifdef __x86_64__
1903  __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1904 #else
1905  __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1906 #endif
1907 }
1908 
1909 __INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
1910 {
1911  unsigned long long retval;
1912  __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
1913  return retval;
1914 }
1915 
1916 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1917 __INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
1918 {
1919  unsigned long retval;
1920  __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
1921  return retval;
1922 }
1923 
1925 {
1926  __asm__ __volatile__("wbinvd" : : : "memory");
1927 }
1928 
1930 {
1931  __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
1932 }
1933 
1935 {
1936  __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
1937 }
1938 
1940 {
1941  __asm__ __volatile__("sgdt %0" : : "m"(*(short*)Destination) : "memory");
1942 }
1943 
1944 /*** Misc operations ***/
1945 
1946 #if !HAS_BUILTIN(_mm_pause)
1948 {
1949  __asm__ __volatile__("pause" : : : "memory");
1950 }
1951 #endif
1952 
1954 {
1955  __asm__ __volatile__("nop");
1956 }
1957 
1958 #ifdef __cplusplus
1959 }
1960 #endif
1961 
1962 #endif /* KJK_INTRIN_X86_H_ */
1963 
1964 /* EOF */
__INTRIN_INLINE void __ud2(void)
Definition: intrin_x86.h:1586
CPPORT Port[4]
Definition: headless.c:34
__INTRIN_INLINE void __writecr4(unsigned int Data)
Definition: intrin_x86.h:1694
__INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
Definition: intrin_x86.h:1508
__INTRIN_INLINE void __stosd(unsigned long *Dest, unsigned long Data, size_t Count)
Definition: intrin_x86.h:750
__INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
Definition: intrin_x86.h:1846
__INTRIN_INLINE void _mm_lfence(void)
Definition: intrin_x86.h:106
__INTRIN_INLINE void __stosb(unsigned char *Dest, unsigned char Data, size_t Count)
Definition: intrin_x86.h:729
__INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
Definition: intrin_x86.h:1176
#define shift
Definition: input.c:1756
__INTRIN_INLINE unsigned long __readcr8(void)
Definition: intrin_x86.h:1735
__INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:968
__INTRIN_INLINE void *__cdecl memcpy(void *dest, const void *source, size_t num)
Definition: intrin_x86.h:74
__INTRIN_INLINE char _InterlockedCompareExchange8(volatile char *Destination, char Exchange, char Comperand)
Definition: intrin_x86.h:330
__INTRIN_INLINE unsigned long __readcr2(void)
Definition: intrin_x86.h:1713
__INTRIN_INLINE void __movsb(unsigned char *Destination, const unsigned char *Source, size_t Count)
Definition: intrin_x86.h:772
__INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
Definition: intrin_x86.h:1887
__INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long *a, long b)
Definition: intrin_x86.h:690
#define __cdecl
Definition: accygwin.h:79
__INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:910
__INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1150
__INTRIN_INLINE void __incfsdword(unsigned long Offset)
Definition: intrin_x86.h:962
__INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
Definition: intrin_x86.h:944
__INTRIN_INLINE unsigned char _BitScanReverse(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1006
__INTRIN_INLINE void __wbinvd(void)
Definition: intrin_x86.h:1924
__INTRIN_INLINE long _InterlockedXor(volatile long *value, long mask)
Definition: intrin_x86.h:581
__INTRIN_INLINE void __incfsword(unsigned long Offset)
Definition: intrin_x86.h:957
__INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
Definition: intrin_x86.h:1238
__INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1440
__INTRIN_INLINE void __halt(void)
Definition: intrin_x86.h:1609
__INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long *Destination, long long Exchange, long long Comperand)
Definition: intrin_x86.h:651
__INTRIN_INLINE void __stosw(unsigned short *Dest, unsigned short Data, size_t Count)
Definition: intrin_x86.h:740
_Inout_ __drv_aliasesMem PSLIST_ENTRY _Inout_ PSLIST_ENTRY _In_ ULONG Count
Definition: exfuncs.h:1223
__INTRIN_INLINE void __lidt(void *Source)
Definition: intrin_x86.h:1929
__INTRIN_INLINE unsigned int __popcnt(unsigned int value)
Definition: intrin_x86.h:1349
GLint GLint GLint GLint GLint x
Definition: gl.h:1548
__INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1281
__INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
Definition: intrin_x86.h:1917
__declspec(noreturn) __INTRIN_INLINE void __fastfail(unsigned int Code)
Definition: intrin_x86.h:1615
__INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
Definition: intrin_x86.h:1317
__INTRIN_INLINE unsigned long __readcr3(void)
Definition: intrin_x86.h:1720
__INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
Definition: intrin_x86.h:926
__INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1462
__INTRIN_INLINE void __movsd(unsigned long *Destination, const unsigned long *Source, size_t Count)
Definition: intrin_x86.h:792
__INTRIN_INLINE long long __emul(int a, int b)
Definition: intrin_x86.h:1377
IN OUT PLONG Addend
Definition: CrNtStubs.h:22
__INTRIN_INLINE void __writecr8(unsigned int Data)
Definition: intrin_x86.h:1700
__INTRIN_INLINE short _InterlockedIncrement16(volatile short *lpAddend)
Definition: intrin_x86.h:621
__INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:915
__INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
Definition: intrin_x86.h:1159
__INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
Definition: intrin_x86.h:1220
__INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
Definition: intrin_x86.h:1335
IN UCHAR Value
Definition: halp.h:394
__INTRIN_INLINE unsigned char _BitScanForward(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:998
__INTRIN_INLINE unsigned char _bittestandset(long *a, long b)
Definition: intrin_x86.h:1087
__INTRIN_INLINE long _InterlockedOr(volatile long *value, long mask)
Definition: intrin_x86.h:524
__INTRIN_INLINE int __cdecl _inp(unsigned short Port)
Definition: intrin_x86.h:1503
__INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
Definition: intrin_x86.h:1302
__INTRIN_INLINE void __cdecl __debugbreak(void)
Definition: intrin_x86.h:1579
__INTRIN_INLINE void __writecr3(unsigned int Data)
Definition: intrin_x86.h:1689
__INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1488
__INTRIN_INLINE unsigned char _bittestandreset(long *a, long b)
Definition: intrin_x86.h:1073
GLenum GLint GLuint mask
Definition: glext.h:6028
__INTRIN_INLINE uintptr_t __readeflags(void)
Definition: intrin_x86.h:1569
_In_ ULONG _In_ ULONG Offset
Definition: ntddpcm.h:101
__INTRIN_INLINE long _InterlockedIncrement(volatile long *lpAddend)
Definition: intrin_x86.h:607
__asm__("\t.globl GetPhys\n" "GetPhys:\t\n" "mflr 0\n\t" "stwu 0,-16(1)\n\t" "mfmsr 5\n\t" "andi. 6,5,0xffef\n\t" "mtmsr 6\n\t" "isync\n\t" "sync\n\t" "lwz 3,0(3)\n\t" "mtmsr 5\n\t" "isync\n\t" "sync\n\t" "lwz 0,0(1)\n\t" "addi 1,1,16\n\t" "mtlr 0\n\t" "blr")
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:1934
__INTRIN_INLINE void _ReadWriteBarrier(void)
Definition: intrin_x86.h:88
#define _ReadBarrier
Definition: intrin_x86.h:95
__INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:976
unsigned int uintptr_t
Definition: crtdefs.h:300
__INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:920
__INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long *const Addend, const long Value)
Definition: intrin_arm.h:359
__INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
Definition: intrin_x86.h:935
__INTRIN_INLINE void __writecr0(unsigned int Data)
Definition: intrin_x86.h:1684
static WCHAR Address[46]
Definition: ping.c:68
__INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
Definition: intrin_x86.h:1419
__INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile *Addend, char Value)
Definition: intrin_x86.h:402
__INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1194
__INTRIN_INLINE short _InterlockedDecrement16(volatile short *lpAddend)
Definition: intrin_x86.h:614
Definition: bufpool.h:45
__INTRIN_INLINE long _InterlockedCompareExchange(volatile long *Destination, long Exchange, long Comperand)
Definition: intrin_x86.h:348
#define _WriteBarrier
Definition: intrin_x86.h:96
#define Code
Definition: deflate.h:80
__INTRIN_INLINE void __nop(void)
Definition: intrin_x86.h:1953
__INTRIN_INLINE long _InterlockedAnd(volatile long *value, long mask)
Definition: intrin_x86.h:467
__INTRIN_INLINE short _InterlockedAnd16(volatile short *value, short mask)
Definition: intrin_x86.h:448
__INTRIN_INLINE void _mm_pause(void)
Definition: intrin_x86.h:1947
GLboolean GLboolean GLboolean b
Definition: glext.h:6204
__INTRIN_INLINE long long __cdecl _abs64(long long value)
Definition: intrin_x86.h:1394
__INTRIN_INLINE void _mm_sfence(void)
Definition: intrin_x86.h:115
__INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
Definition: intrin_x86.h:1267
__INTRIN_INLINE char _InterlockedXor8(volatile char *value, char mask)
Definition: intrin_x86.h:543
__INTRIN_INLINE void * _InterlockedCompareExchangePointer(void *volatile *Destination, void *Exchange, void *Comperand)
Definition: intrin_x86.h:357
__INTRIN_INLINE long _InterlockedDecrement(volatile long *lpAddend)
Definition: intrin_x86.h:600
__INTRIN_INLINE void * _InterlockedExchangePointer(void *volatile *Target, void *Value)
Definition: intrin_x86.h:393
__INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
Definition: intrin_x86.h:1185
static const UCHAR Index[8]
Definition: usbohci.c:18
__INTRIN_INLINE void __cdecl _enable(void)
Definition: intrin_x86.h:1604
__INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
Definition: intrin_x86.h:1900
GLuint GLuint num
Definition: glext.h:9618
__INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
Definition: intrin_x86.h:1539
_In_ PUNICODE_STRING _Inout_ PUNICODE_STRING Destination
Definition: rtlfuncs.h:2937
__INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
Definition: intrin_x86.h:1513
_Must_inspect_result_ typedef _In_ ULONG _In_ BOOLEAN Target
Definition: iotypes.h:1088
__INTRIN_INLINE char _InterlockedExchange8(volatile char *Target, char Value)
Definition: intrin_x86.h:366
__INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1498
__INTRIN_INLINE long _InterlockedExchangeAdd(volatile long *Addend, long Value)
Definition: intrin_x86.h:420
__INTRIN_INLINE void __int2c(void)
Definition: intrin_x86.h:1593
__INTRIN_INLINE long _InterlockedExchange(volatile long *Target, long Value)
Definition: intrin_x86.h:384
__INTRIN_INLINE short _InterlockedCompareExchange16(volatile short *Destination, short Exchange, short Comperand)
Definition: intrin_x86.h:339
#define __INTRIN_INLINE
Definition: intrin.h:39
__INTRIN_INLINE void __writeeflags(uintptr_t Value)
Definition: intrin_x86.h:1564
__INTRIN_INLINE unsigned long long __rdtsc(void)
Definition: intrin_x86.h:1550
__INTRIN_INLINE void __cdecl _disable(void)
Definition: intrin_x86.h:1599
__INTRIN_INLINE unsigned int __readdr(unsigned int reg)
Definition: intrin_x86.h:1813
__INTRIN_INLINE char _InterlockedOr8(volatile char *value, char mask)
Definition: intrin_x86.h:486
__INTRIN_INLINE unsigned long __readcr0(void)
Definition: intrin_x86.h:1706
__INTRIN_INLINE unsigned long __readcr4(void)
Definition: intrin_x86.h:1727
GLsizei const GLfloat * value
Definition: glext.h:6069
__INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1451
__INTRIN_INLINE void _sgdt(void *Destination)
Definition: intrin_x86.h:1939
__INTRIN_INLINE unsigned long __indword(unsigned short Port)
Definition: intrin_x86.h:1433
__INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
Definition: intrin_x86.h:1386
#define local
Definition: zutil.h:30
__INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
Definition: intrin_x86.h:1909
__INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
Definition: intrin_x86.h:1342
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1544
__INTRIN_INLINE char _InterlockedAnd8(volatile char *value, char mask)
Definition: intrin_x86.h:429
#define byte(x, n)
Definition: tomcrypt.h:118
__INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:984
CPUINFO CPUInfo[]
Definition: parse.c:231
__INTRIN_INLINE void __incfsbyte(unsigned long Offset)
Definition: intrin_x86.h:952
__INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
Definition: intrin_x86.h:1524
__INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1253
static int reg
Definition: i386-dis.c:1275
void *__cdecl memmove(void *dest, const void *source, size_t num)
Definition: memmove.c:4
GLint GLint GLint GLint GLint GLint y
Definition: gl.h:1548
__INTRIN_INLINE short _InterlockedOr16(volatile short *value, short mask)
Definition: intrin_x86.h:505
const WCHAR * word
Definition: lex.c:36
__INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
Definition: intrin_x86.h:1518
__INTRIN_INLINE short _InterlockedExchange16(volatile short *Target, short Value)
Definition: intrin_x86.h:375
__INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short *Addend, short Value)
Definition: intrin_x86.h:411
__INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1141
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1879
__INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
Definition: intrin_x86.h:1229
__INTRIN_INLINE unsigned char _bittest(const long *a, long b)
Definition: intrin_x86.h:1015
__INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1203
__INTRIN_INLINE short _InterlockedXor16(volatile short *value, short mask)
Definition: intrin_x86.h:562
__INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
Definition: intrin_x86.h:1295
__INTRIN_INLINE void _mm_mfence(void)
Definition: intrin_x86.h:99
__INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
Definition: intrin_x86.h:1530
_In_ UINT _In_ UINT _In_ PNDIS_PACKET Source
Definition: ndis.h:3167
static char * dest
Definition: rtl.c:135
__INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1493
GLboolean GLboolean GLboolean GLboolean a
Definition: glext.h:6204
__INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
Definition: intrin_x86.h:1478
__INTRIN_INLINE unsigned short __inword(unsigned short Port)
Definition: intrin_x86.h:1426
__INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
Definition: intrin_x86.h:1473
__INTRIN_INLINE void __movsw(unsigned short *Destination, const unsigned short *Source, size_t Count)
Definition: intrin_x86.h:782
__INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
Definition: intrin_x86.h:1483
__INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long *a, long b)
Definition: intrin_x86.h:708
__INTRIN_INLINE unsigned char _bittestandcomplement(long *a, long b)
Definition: intrin_x86.h:1059
__INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
Definition: intrin_x86.h:1356