ReactOS  0.4.15-dev-4853-g3a72a52
intrin_x86.h
Go to the documentation of this file.
1 /*
2  Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3  Microsoft Visual C++ functions. Originally developed for the ReactOS
4  (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5  projects.
6 
7  Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8 
9  Permission is hereby granted, free of charge, to any person obtaining a
10  copy of this software and associated documentation files (the "Software"),
11  to deal in the Software without restriction, including without limitation
12  the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  and/or sell copies of the Software, and to permit persons to whom the
14  Software is furnished to do so, subject to the following conditions:
15 
16  The above copyright notice and this permission notice shall be included in
17  all copies or substantial portions of the Software.
18 
19  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  DEALINGS IN THE SOFTWARE.
26 */
27 
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30 
31 /*
32  FIXME: review all "memory" clobbers, add/remove to match Visual C++
33  behavior: some "obvious" memory barriers are not present in the Visual C++
34  implementation - e.g. __stosX; on the other hand, some memory barriers that
35  *are* present could have been missed
36 */
37 
38 /*
39  NOTE: this is a *compatibility* header. Some functions may look wrong at
40  first, but they're only "as wrong" as they would be on Visual C++. Our
41  priority is compatibility
42 
43  NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44  constraints and the uses of __volatile__ out of my... hat. Do not touch
45  them. I hate cargo cult programming
46 
47  NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48  barriers aren't there in Visual C++ (e.g. __stosX)
49 
50  NOTE: review all intrinsics with a return value, add/remove __volatile__
51  where necessary. If an intrinsic whose value is ignored generates a no-op
52  under Visual C++, __volatile__ must be omitted; if it always generates code
53  (for example, if it has side effects), __volatile__ must be specified. GCC
54  will only optimize out non-volatile asm blocks with outputs, so input-only
55  blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56  and follow Visual C++ behavior
57 
58  NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59  atomic operations. Test the version like this:
60 
61  #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62  ...
63 
64  Pay attention to the type of barrier. Make it match with what Visual C++
65  would use in the same case
66 */
67 
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71 
72 /*** memcopy must be memmove ***/
73 void* __cdecl memmove(void* dest, const void* source, size_t num);
74 __INTRIN_INLINE void* __cdecl memcpy(void* dest, const void* source, size_t num)
75 {
76  return memmove(dest, source, num);
77 }
78 
79 
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
84 
85 /*** Memory barriers ***/
86 
87 #if !HAS_BUILTIN(_ReadWriteBarrier)
89 {
90  __asm__ __volatile__("" : : : "memory");
91 }
92 #endif
93 
94 /* GCC only supports full barriers */
95 #define _ReadBarrier _ReadWriteBarrier
96 #define _WriteBarrier _ReadWriteBarrier
97 
98 #if !HAS_BUILTIN(_mm_mfence)
100 {
101  __asm__ __volatile__("mfence" : : : "memory");
102 }
103 #endif
104 
105 #if !HAS_BUILTIN(_mm_lfence)
107 {
108  _ReadBarrier();
109  __asm__ __volatile__("lfence");
110  _ReadBarrier();
111 }
112 #endif
113 
114 #if defined(__x86_64__) && !HAS_BUILTIN(__faststorefence)
115 __INTRIN_INLINE void __faststorefence(void)
116 {
117  long local;
118  __asm__ __volatile__("lock; orl $0, %0;" : : "m"(local));
119 }
120 #endif
121 
122 
123 /*** Atomic operations ***/
124 
125 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
126 
127 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
128 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
129 {
130  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
131 }
132 #endif
133 
134 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
135 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
136 {
137  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
138 }
139 #endif
140 
141 #if !HAS_BUILTIN(_InterlockedCompareExchange)
142 __INTRIN_INLINE long __cdecl _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
143 {
144  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
145 }
146 #endif
147 
148 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
149 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
150 {
151  return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
152 }
153 #endif
154 
155 #if !HAS_BUILTIN(_InterlockedExchange8)
156 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
157 {
158  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
159  __sync_synchronize();
160  return __sync_lock_test_and_set(Target, Value);
161 }
162 #endif
163 
164 #if !HAS_BUILTIN(_InterlockedExchange16)
165 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
166 {
167  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
168  __sync_synchronize();
169  return __sync_lock_test_and_set(Target, Value);
170 }
171 #endif
172 
173 #if !HAS_BUILTIN(_InterlockedExchange)
174 __INTRIN_INLINE long __cdecl _InterlockedExchange(volatile long * Target, long Value)
175 {
176  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
177  __sync_synchronize();
178  return __sync_lock_test_and_set(Target, Value);
179 }
180 #endif
181 
182 #if !HAS_BUILTIN(_InterlockedExchangePointer)
183 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
184 {
185  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
186  __sync_synchronize();
187  return (void *)__sync_lock_test_and_set(Target, Value);
188 }
189 #endif
190 
191 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedExchange64)
192 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * Target, long long Value)
193 {
194  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
195  __sync_synchronize();
196  return __sync_lock_test_and_set(Target, Value);
197 }
198 #endif
199 
200 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
201 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
202 {
203  return __sync_fetch_and_add(Addend, Value);
204 }
205 #endif
206 
207 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
208 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
209 {
210  return __sync_fetch_and_add(Addend, Value);
211 }
212 #endif
213 
214 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
215 __INTRIN_INLINE long __cdecl _InterlockedExchangeAdd(volatile long * Addend, long Value)
216 {
217  return __sync_fetch_and_add(Addend, Value);
218 }
219 #endif
220 
221 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedExchangeAdd64)
222 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * Addend, long long Value)
223 {
224  return __sync_fetch_and_add(Addend, Value);
225 }
226 #endif
227 
228 #if !HAS_BUILTIN(_InterlockedAnd8)
229 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
230 {
231  return __sync_fetch_and_and(value, mask);
232 }
233 #endif
234 
235 #if !HAS_BUILTIN(_InterlockedAnd16)
236 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
237 {
238  return __sync_fetch_and_and(value, mask);
239 }
240 #endif
241 
242 #if !HAS_BUILTIN(_InterlockedAnd)
243 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
244 {
245  return __sync_fetch_and_and(value, mask);
246 }
247 #endif
248 
249 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedAnd64)
250 __INTRIN_INLINE long long _InterlockedAnd64(volatile long long * value, long long mask)
251 {
252  return __sync_fetch_and_and(value, mask);
253 }
254 #endif
255 
256 #if !HAS_BUILTIN(_InterlockedOr8)
257 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
258 {
259  return __sync_fetch_and_or(value, mask);
260 }
261 #endif
262 
263 #if !HAS_BUILTIN(_InterlockedOr16)
264 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
265 {
266  return __sync_fetch_and_or(value, mask);
267 }
268 #endif
269 
270 #if !HAS_BUILTIN(_InterlockedOr)
271 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
272 {
273  return __sync_fetch_and_or(value, mask);
274 }
275 #endif
276 
277 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedOr64)
278 __INTRIN_INLINE long long _InterlockedOr64(volatile long long * value, long long mask)
279 {
280  return __sync_fetch_and_or(value, mask);
281 }
282 #endif
283 
284 #if !HAS_BUILTIN(_InterlockedXor8)
285 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
286 {
287  return __sync_fetch_and_xor(value, mask);
288 }
289 #endif
290 
291 #if !HAS_BUILTIN(_InterlockedXor16)
292 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
293 {
294  return __sync_fetch_and_xor(value, mask);
295 }
296 #endif
297 
298 #if !HAS_BUILTIN(_InterlockedXor)
299 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
300 {
301  return __sync_fetch_and_xor(value, mask);
302 }
303 #endif
304 
305 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedXor64)
306 __INTRIN_INLINE long long _InterlockedXor64(volatile long long * value, long long mask)
307 {
308  return __sync_fetch_and_xor(value, mask);
309 }
310 #endif
311 
312 #if !HAS_BUILTIN(_InterlockedDecrement)
313 __INTRIN_INLINE long __cdecl _InterlockedDecrement(volatile long * lpAddend)
314 {
315  return __sync_sub_and_fetch(lpAddend, 1);
316 }
317 #endif
318 
319 #if !HAS_BUILTIN(_InterlockedIncrement)
320 __INTRIN_INLINE long __cdecl _InterlockedIncrement(volatile long * lpAddend)
321 {
322  return __sync_add_and_fetch(lpAddend, 1);
323 }
324 #endif
325 
326 #if !HAS_BUILTIN(_InterlockedDecrement16)
327 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
328 {
329  return __sync_sub_and_fetch(lpAddend, 1);
330 }
331 #endif
332 
333 #if !HAS_BUILTIN(_InterlockedIncrement16)
334 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
335 {
336  return __sync_add_and_fetch(lpAddend, 1);
337 }
338 #endif
339 
340 #if defined(__x86_64__)
341 #if !HAS_BUILTIN(_InterlockedDecrement64)
342 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
343 {
344  return __sync_sub_and_fetch(lpAddend, 1);
345 }
346 #endif
347 
348 #if !HAS_BUILTIN(_InterlockedIncrement64)
349 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
350 {
351  return __sync_add_and_fetch(lpAddend, 1);
352 }
353 #endif
354 #endif
355 
356 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
357 
358 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
359 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
360 {
361  char retval = Comperand;
362  __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
363  return retval;
364 }
365 #endif
366 
367 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
368 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
369 {
370  short retval = Comperand;
371  __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
372  return retval;
373 }
374 #endif
375 
376 #if !HAS_BUILTIN(_InterlockedCompareExchange)
377 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
378 {
379  long retval = Comperand;
380  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
381  return retval;
382 }
383 #endif
384 
385 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
386 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
387 {
388  void * retval = (void *)Comperand;
389  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
390  return retval;
391 }
392 #endif
393 
394 #if !HAS_BUILTIN(_InterlockedExchange8)
396 {
397  char retval = Value;
398  __asm__("xchgb %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
399  return retval;
400 }
401 #endif
402 
403 #if !HAS_BUILTIN(_InterlockedExchange16)
404 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
405 {
406  short retval = Value;
407  __asm__("xchgw %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
408  return retval;
409 }
410 #endif
411 
412 #if !HAS_BUILTIN(_InterlockedExchange)
414 {
415  long retval = Value;
416  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
417  return retval;
418 }
419 #endif
420 
421 #if !HAS_BUILTIN(_InterlockedExchangePointer)
422 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
423 {
424  void * retval = Value;
425  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
426  return retval;
427 }
428 #endif
429 
430 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
432 {
433  char retval = Value;
434  __asm__("lock; xaddb %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
435  return retval;
436 }
437 #endif
438 
439 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
441 {
442  short retval = Value;
443  __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
444  return retval;
445 }
446 #endif
447 
448 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
450 {
451  long retval = Value;
452  __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
453  return retval;
454 }
455 #endif
456 
457 #if !HAS_BUILTIN(_InterlockedAnd8)
458 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
459 {
460  char x;
461  char y;
462 
463  y = *value;
464 
465  do
466  {
467  x = y;
469  }
470  while(y != x);
471 
472  return y;
473 }
474 #endif
475 
476 #if !HAS_BUILTIN(_InterlockedAnd16)
477 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
478 {
479  short x;
480  short y;
481 
482  y = *value;
483 
484  do
485  {
486  x = y;
488  }
489  while(y != x);
490 
491  return y;
492 }
493 #endif
494 
495 #if !HAS_BUILTIN(_InterlockedAnd)
496 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
497 {
498  long x;
499  long y;
500 
501  y = *value;
502 
503  do
504  {
505  x = y;
507  }
508  while(y != x);
509 
510  return y;
511 }
512 #endif
513 
514 #if !HAS_BUILTIN(_InterlockedOr8)
515 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
516 {
517  char x;
518  char y;
519 
520  y = *value;
521 
522  do
523  {
524  x = y;
526  }
527  while(y != x);
528 
529  return y;
530 }
531 #endif
532 
533 #if !HAS_BUILTIN(_InterlockedOr16)
534 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
535 {
536  short x;
537  short y;
538 
539  y = *value;
540 
541  do
542  {
543  x = y;
545  }
546  while(y != x);
547 
548  return y;
549 }
550 #endif
551 
552 #if !HAS_BUILTIN(_InterlockedOr)
553 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
554 {
555  long x;
556  long y;
557 
558  y = *value;
559 
560  do
561  {
562  x = y;
564  }
565  while(y != x);
566 
567  return y;
568 }
569 #endif
570 
571 #if !HAS_BUILTIN(_InterlockedXor8)
572 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
573 {
574  char x;
575  char y;
576 
577  y = *value;
578 
579  do
580  {
581  x = y;
583  }
584  while(y != x);
585 
586  return y;
587 }
588 #endif
589 
590 #if !HAS_BUILTIN(_InterlockedXor16)
591 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
592 {
593  short x;
594  short y;
595 
596  y = *value;
597 
598  do
599  {
600  x = y;
602  }
603  while(y != x);
604 
605  return y;
606 }
607 #endif
608 
609 #if !HAS_BUILTIN(_InterlockedXor)
610 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
611 {
612  long x;
613  long y;
614 
615  y = *value;
616 
617  do
618  {
619  x = y;
621  }
622  while(y != x);
623 
624  return y;
625 }
626 #endif
627 
628 #if !HAS_BUILTIN(_InterlockedDecrement)
629 __INTRIN_INLINE long _InterlockedDecrement(volatile long * lpAddend)
630 {
631  return _InterlockedExchangeAdd(lpAddend, -1) - 1;
632 }
633 #endif
634 
635 #if !HAS_BUILTIN(_InterlockedIncrement)
636 __INTRIN_INLINE long _InterlockedIncrement(volatile long * lpAddend)
637 {
638  return _InterlockedExchangeAdd(lpAddend, 1) + 1;
639 }
640 #endif
641 
642 #if !HAS_BUILTIN(_InterlockedDecrement16)
643 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
644 {
645  return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
646 }
647 #endif
648 
649 #if !HAS_BUILTIN(_InterlockedIncrement16)
650 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
651 {
652  return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
653 }
654 #endif
655 
656 #if defined(__x86_64__)
657 #if !HAS_BUILTIN(_InterlockedDecrement64)
658 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
659 {
660  return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
661 }
662 #endif
663 
664 #if !HAS_BUILTIN(_InterlockedIncrement64)
665 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
666 {
667  return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
668 }
669 #endif
670 #endif
671 
672 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
673 
674 #if !HAS_BUILTIN(_InterlockedCompareExchange64)
675 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
676 
677 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
678 {
679  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
680 }
681 
682 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
683 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
684 {
685  long long retval = Comperand;
686 
687  __asm__
688  (
689  "lock; cmpxchg8b %[Destination]" :
690  [retval] "+A" (retval) :
691  [Destination] "m" (*Destination),
692  "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
693  "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
694  "memory"
695  );
696 
697  return retval;
698 }
699 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
700 #endif /* !HAS_BUILTIN(_InterlockedCompareExchange64) */
701 
702 #ifdef __i386__
703 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * Addend, long Value)
704 {
705  __asm__
706  (
707  "lock; addl %[Value], %[Lo32];"
708  "jae LABEL%=;"
709  "lock; adcl $0, %[Hi32];"
710  "LABEL%=:;" :
711  [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
712  [Value] "ir" (Value) :
713  "memory"
714  );
715 
716  return Value;
717 }
718 #endif /* __i386__ */
719 
720 #if !HAS_BUILTIN(_interlockedbittestandreset)
721 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, long b)
722 {
723  unsigned char retval;
724  __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
725  return retval;
726 }
727 #endif
728 
729 #if defined(__x86_64__) && !HAS_BUILTIN(_interlockedbittestandreset64)
730 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, long long b)
731 {
732  unsigned char retval;
733  __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
734  return retval;
735 }
736 
737 #endif
738 
739 #if !HAS_BUILTIN(_interlockedbittestandset)
740 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, long b)
741 {
742  unsigned char retval;
743  __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
744  return retval;
745 }
746 #endif
747 
748 #if defined(__x86_64__) && !HAS_BUILTIN(_interlockedbittestandset64)
749 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, long long b)
750 {
751  unsigned char retval;
752  __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
753  return retval;
754 }
755 #endif
756 
757 /*** String operations ***/
758 
759 #if !HAS_BUILTIN(__stosb)
760 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
761 __INTRIN_INLINE void __stosb(unsigned char * Dest, unsigned char Data, size_t Count)
762 {
763  __asm__ __volatile__
764  (
765  "rep; stosb" :
766  [Dest] "=D" (Dest), [Count] "=c" (Count) :
767  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
768  );
769 }
770 #endif
771 
772 __INTRIN_INLINE void __stosw(unsigned short * Dest, unsigned short Data, size_t Count)
773 {
774  __asm__ __volatile__
775  (
776  "rep; stosw" :
777  [Dest] "=D" (Dest), [Count] "=c" (Count) :
778  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
779  );
780 }
781 
782 __INTRIN_INLINE void __stosd(unsigned long * Dest, unsigned long Data, size_t Count)
783 {
784  __asm__ __volatile__
785  (
786  "rep; stosl" :
787  [Dest] "=D" (Dest), [Count] "=c" (Count) :
788  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
789  );
790 }
791 
792 #ifdef __x86_64__
793 __INTRIN_INLINE void __stosq(unsigned long long * Dest, unsigned long long Data, size_t Count)
794 {
795  __asm__ __volatile__
796  (
797  "rep; stosq" :
798  [Dest] "=D" (Dest), [Count] "=c" (Count) :
799  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
800  );
801 }
802 #endif
803 
804 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
805 {
806  __asm__ __volatile__
807  (
808  "rep; movsb" :
809  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
810  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
811  );
812 }
813 
814 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
815 {
816  __asm__ __volatile__
817  (
818  "rep; movsw" :
819  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
820  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
821  );
822 }
823 
824 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
825 {
826  __asm__ __volatile__
827  (
828  "rep; movsd" :
829  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
830  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
831  );
832 }
833 
834 #ifdef __x86_64__
835 __INTRIN_INLINE void __movsq(unsigned long long * Destination, const unsigned long long * Source, size_t Count)
836 {
837  __asm__ __volatile__
838  (
839  "rep; movsq" :
840  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
841  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
842  );
843 }
844 #endif
845 
846 #if defined(__x86_64__)
847 
848 /*** GS segment addressing ***/
849 
850 __INTRIN_INLINE void __writegsbyte(unsigned long Offset, unsigned char Data)
851 {
852  __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
853 }
854 
855 __INTRIN_INLINE void __writegsword(unsigned long Offset, unsigned short Data)
856 {
857  __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
858 }
859 
860 __INTRIN_INLINE void __writegsdword(unsigned long Offset, unsigned long Data)
861 {
862  __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
863 }
864 
865 __INTRIN_INLINE void __writegsqword(unsigned long Offset, unsigned long long Data)
866 {
867  __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
868 }
869 
870 #if !HAS_BUILTIN(__readgsbyte)
871 __INTRIN_INLINE unsigned char __readgsbyte(unsigned long Offset)
872 {
873  unsigned char value;
874  __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
875  return value;
876 }
877 #endif
878 
879 #if !HAS_BUILTIN(__readgsword)
880 __INTRIN_INLINE unsigned short __readgsword(unsigned long Offset)
881 {
882  unsigned short value;
883  __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
884  return value;
885 }
886 #endif
887 
888 #if !HAS_BUILTIN(__readgsdword)
889 __INTRIN_INLINE unsigned long __readgsdword(unsigned long Offset)
890 {
891  unsigned long value;
892  __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
893  return value;
894 }
895 #endif
896 
897 #if !HAS_BUILTIN(__readgsqword)
898 __INTRIN_INLINE unsigned long long __readgsqword(unsigned long Offset)
899 {
900  unsigned long long value;
901  __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
902  return value;
903 }
904 #endif
905 
906 __INTRIN_INLINE void __incgsbyte(unsigned long Offset)
907 {
908  __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
909 }
910 
911 __INTRIN_INLINE void __incgsword(unsigned long Offset)
912 {
913  __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
914 }
915 
916 __INTRIN_INLINE void __incgsdword(unsigned long Offset)
917 {
918  __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
919 }
920 
921 __INTRIN_INLINE void __incgsqword(unsigned long Offset)
922 {
923  __asm__ __volatile__("incq %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
924 }
925 
926 __INTRIN_INLINE void __addgsbyte(unsigned long Offset, unsigned char Data)
927 {
928  __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
929 }
930 
931 __INTRIN_INLINE void __addgsword(unsigned long Offset, unsigned short Data)
932 {
933  __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
934 }
935 
936 __INTRIN_INLINE void __addgsdword(unsigned long Offset, unsigned long Data)
937 {
938  __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
939 }
940 
941 __INTRIN_INLINE void __addgsqword(unsigned long Offset, unsigned long long Data)
942 {
943  __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
944 }
945 
946 #else /* defined(__x86_64__) */
947 
948 /*** FS segment addressing ***/
949 
950 __INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
951 {
952  __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
953 }
954 
955 __INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
956 {
957  __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
958 }
959 
960 __INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
961 {
962  __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
963 }
964 
965 #if !HAS_BUILTIN(__readfsbyte)
966 __INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
967 {
968  unsigned char value;
969  __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
970  return value;
971 }
972 #endif
973 
974 #if !HAS_BUILTIN(__readfsword)
975 __INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
976 {
977  unsigned short value;
978  __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
979  return value;
980 }
981 #endif
982 
983 #if !HAS_BUILTIN(__readfsdword)
984 __INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
985 {
986  unsigned long value;
987  __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
988  return value;
989 }
990 #endif
991 
992 __INTRIN_INLINE void __incfsbyte(unsigned long Offset)
993 {
994  __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
995 }
996 
997 __INTRIN_INLINE void __incfsword(unsigned long Offset)
998 {
999  __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
1000 }
1001 
1003 {
1004  __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
1005 }
1006 
1007 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
1008 __INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
1009 {
1010  if(!__builtin_constant_p(Offset))
1011  __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1012  else
1013  __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1014 }
1015 
1016 __INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
1017 {
1018  if(!__builtin_constant_p(Offset))
1019  __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1020  else
1021  __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1022 }
1023 
1024 __INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
1025 {
1026  if(!__builtin_constant_p(Offset))
1027  __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1028  else
1029  __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1030 }
1031 
1032 #endif /* defined(__x86_64__) */
1033 
1034 
1035 /*** Bit manipulation ***/
1036 
1037 #if !HAS_BUILTIN(_BitScanForward)
1038 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * Index, unsigned long Mask)
1039 {
1040  __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1041  return Mask ? 1 : 0;
1042 }
1043 #endif
1044 
1045 #if !HAS_BUILTIN(_BitScanReverse)
1046 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * Index, unsigned long Mask)
1047 {
1048  __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1049  return Mask ? 1 : 0;
1050 }
1051 #endif
1052 
1053 #if !HAS_BUILTIN(_bittest)
1054 /* NOTE: again, the bizarre implementation follows Visual C++ */
1055 __INTRIN_INLINE unsigned char _bittest(const long * a, long b)
1056 {
1057  unsigned char retval;
1058 
1059  if(__builtin_constant_p(b))
1060  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
1061  else
1062  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1063 
1064  return retval;
1065 }
1066 #endif
1067 
1068 #ifdef __x86_64__
1069 #if !HAS_BUILTIN(_BitScanForward64)
1070 __INTRIN_INLINE unsigned char _BitScanForward64(unsigned long * Index, unsigned long long Mask)
1071 {
1072  unsigned long long Index64;
1073  __asm__("bsfq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1074  *Index = Index64;
1075  return Mask ? 1 : 0;
1076 }
1077 #endif
1078 
1079 #if !HAS_BUILTIN(_BitScanReverse64)
1080 __INTRIN_INLINE unsigned char _BitScanReverse64(unsigned long * Index, unsigned long long Mask)
1081 {
1082  unsigned long long Index64;
1083  __asm__("bsrq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1084  *Index = Index64;
1085  return Mask ? 1 : 0;
1086 }
1087 #endif
1088 
1089 #if !HAS_BUILTIN(_bittest64)
1090 __INTRIN_INLINE unsigned char _bittest64(const long long * a, long long b)
1091 {
1092  unsigned char retval;
1093 
1094  if(__builtin_constant_p(b))
1095  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
1096  else
1097  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1098 
1099  return retval;
1100 }
1101 #endif
1102 #endif
1103 
1104 #if !HAS_BUILTIN(_bittestandcomplement)
1105 __INTRIN_INLINE unsigned char _bittestandcomplement(long * a, long b)
1106 {
1107  unsigned char retval;
1108 
1109  if(__builtin_constant_p(b))
1110  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1111  else
1112  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1113 
1114  return retval;
1115 }
1116 #endif
1117 
1118 #if !HAS_BUILTIN(_bittestandreset)
1119 __INTRIN_INLINE unsigned char _bittestandreset(long * a, long b)
1120 {
1121  unsigned char retval;
1122 
1123  if(__builtin_constant_p(b))
1124  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1125  else
1126  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1127 
1128  return retval;
1129 }
1130 #endif
1131 
1132 #if !HAS_BUILTIN(_bittestandset)
1133 __INTRIN_INLINE unsigned char _bittestandset(long * a, long b)
1134 {
1135  unsigned char retval;
1136 
1137  if(__builtin_constant_p(b))
1138  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1139  else
1140  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1141 
1142  return retval;
1143 }
1144 #endif
1145 
1146 #ifdef __x86_64__
1147 #if !HAS_BUILTIN(_bittestandset64)
1148 __INTRIN_INLINE unsigned char _bittestandset64(long long * a, long long b)
1149 {
1150  unsigned char retval;
1151 
1152  if(__builtin_constant_p(b))
1153  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1154  else
1155  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1156 
1157  return retval;
1158 }
1159 #endif
1160 
1161 #if !HAS_BUILTIN(_bittestandreset64)
1162 __INTRIN_INLINE unsigned char _bittestandreset64(long long * a, long long b)
1163 {
1164  unsigned char retval;
1165 
1166  if(__builtin_constant_p(b))
1167  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1168  else
1169  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1170 
1171  return retval;
1172 }
1173 #endif
1174 
1175 #if !HAS_BUILTIN(_bittestandcomplement64)
1176 __INTRIN_INLINE unsigned char _bittestandcomplement64(long long * a, long long b)
1177 {
1178  unsigned char retval;
1179 
1180  if(__builtin_constant_p(b))
1181  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1182  else
1183  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1184 
1185  return retval;
1186 }
1187 #endif
1188 #endif /* __x86_64__ */
1189 
1190 #if !HAS_BUILTIN(_rotl8)
1191 __INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
1192 {
1193  unsigned char retval;
1194  __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1195  return retval;
1196 }
1197 #endif
1198 
1199 #if !HAS_BUILTIN(_rotl16)
1200 __INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
1201 {
1202  unsigned short retval;
1203  __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1204  return retval;
1205 }
1206 #endif
1207 
1208 #if !HAS_BUILTIN(_rotl)
1209 __INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
1210 {
1211  unsigned int retval;
1212  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1213  return retval;
1214 }
1215 #endif
1216 
1217 #if !HAS_BUILTIN(_rotl64)
1218 #ifdef __x86_64__
1219 __INTRIN_INLINE unsigned long long _rotl64(unsigned long long value, int shift)
1220 {
1221  unsigned long long retval;
1222  __asm__("rolq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1223  return retval;
1224 }
1225 #else /* __x86_64__ */
1226 __INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
1227 {
1228  /* FIXME: this is probably not optimal */
1229  return (value << shift) | (value >> (64 - shift));
1230 }
1231 #endif /* __x86_64__ */
1232 #endif /* !HAS_BUILTIN(_rotl64) */
1233 
1234 #if !HAS_BUILTIN(_rotr)
1235 __INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
1236 {
1237  unsigned int retval;
1238  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1239  return retval;
1240 }
1241 #endif
1242 
1243 #if !HAS_BUILTIN(_rotr8)
1244 __INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
1245 {
1246  unsigned char retval;
1247  __asm__("rorb %b[shift], %b[retval]" : [retval] "=qm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1248  return retval;
1249 }
1250 #endif
1251 
1252 #if !HAS_BUILTIN(_rotr16)
1253 __INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
1254 {
1255  unsigned short retval;
1256  __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1257  return retval;
1258 }
1259 #endif
1260 
1261 #if !HAS_BUILTIN(_rotr64)
1262 #ifdef __x86_64__
1263 __INTRIN_INLINE unsigned long long _rotr64(unsigned long long value, int shift)
1264 {
1265  unsigned long long retval;
1266  __asm__("rorq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1267  return retval;
1268 }
1269 #else /* __x86_64__ */
1270 __INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
1271 {
1272  /* FIXME: this is probably not optimal */
1273  return (value >> shift) | (value << (64 - shift));
1274 }
1275 #endif /* __x86_64__ */
1276 #endif /* !HAS_BUILTIN(_rotr64) */
1277 
1278 #if !HAS_BUILTIN(_lrotl)
1279 __INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
1280 {
1281  unsigned long retval;
1282  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1283  return retval;
1284 }
1285 #endif
1286 
1287 #if !HAS_BUILTIN(_lrotr)
1288 __INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
1289 {
1290  unsigned long retval;
1291  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1292  return retval;
1293 }
1294 #endif
1295 
1296 #ifdef __x86_64__
1297 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1298 {
1299  unsigned long long retval;
1300  unsigned char shift = Bit & 0x3F;
1301 
1302  __asm__
1303  (
1304  "shlq %[shift], %[Mask]" : "=r"(retval) : [Mask] "0"(Mask), [shift] "c"(shift)
1305  );
1306 
1307  return retval;
1308 }
1309 
1310 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1311 {
1312  long long retval;
1313  unsigned char shift = Bit & 0x3F;
1314 
1315  __asm__
1316  (
1317  "sarq %[shift], %[Mask]" : "=r"(retval) : [Mask] "0"(Mask), [shift] "c"(shift)
1318  );
1319 
1320  return retval;
1321 }
1322 
1323 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1324 {
1325  long long retval;
1326  unsigned char shift = Bit & 0x3F;
1327 
1328  __asm__
1329  (
1330  "shrq %[shift], %[Mask]" : "=r"(retval) : [Mask] "0"(Mask), [shift] "c"(shift)
1331  );
1332 
1333  return retval;
1334 }
1335 #else
1336 /*
1337  NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1338  constraint (edx:eax) for the Mask argument, because it's the only way GCC
1339  can pass 64-bit operands around - passing the two 32 bit parts separately
1340  just confuses it. Also we declare Bit as an int and then truncate it to
1341  match Visual C++ behavior
1342 */
1343 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1344 {
1345  unsigned long long retval = Mask;
1346 
1347  __asm__
1348  (
1349  "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1350  "+A" (retval) :
1351  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1352  );
1353 
1354  return retval;
1355 }
1356 
1357 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1358 {
1359  long long retval = Mask;
1360 
1361  __asm__
1362  (
1363  "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1364  "+A" (retval) :
1365  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1366  );
1367 
1368  return retval;
1369 }
1370 
1371 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1372 {
1373  unsigned long long retval = Mask;
1374 
1375  __asm__
1376  (
1377  "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1378  "+A" (retval) :
1379  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1380  );
1381 
1382  return retval;
1383 }
1384 #endif
1385 
1386 __INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
1387 {
1388  unsigned short retval;
1389  __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
1390  return retval;
1391 }
1392 
1393 __INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
1394 {
1395  unsigned long retval;
1396  __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1397  return retval;
1398 }
1399 
1400 #ifdef __x86_64__
1401 __INTRIN_INLINE unsigned long long _byteswap_uint64(unsigned long long value)
1402 {
1403  unsigned long long retval;
1404  __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1405  return retval;
1406 }
1407 #else
1408 __INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
1409 {
1410  union {
1411  unsigned long long int64part;
1412  struct {
1413  unsigned long lowpart;
1414  unsigned long hipart;
1415  };
1416  } retval;
1417  retval.int64part = value;
1418  __asm__("bswapl %[lowpart]\n"
1419  "bswapl %[hipart]\n"
1420  : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
1421  return retval.int64part;
1422 }
1423 #endif
1424 
1425 #if !HAS_BUILTIN(__lzcnt)
1426 __INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
1427 {
1428  return __builtin_clz(value);
1429 }
1430 #endif
1431 
1432 #if !HAS_BUILTIN(__lzcnt16)
1433 __INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
1434 {
1435  return __builtin_clz(value);
1436 }
1437 #endif
1438 
1439 #if !HAS_BUILTIN(__popcnt)
1440 __INTRIN_INLINE unsigned int __popcnt(unsigned int value)
1441 {
1442  return __builtin_popcount(value);
1443 }
1444 #endif
1445 
1446 #if !HAS_BUILTIN(__popcnt16)
1447 __INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
1448 {
1449  return __builtin_popcount(value);
1450 }
1451 #endif
1452 
1453 #ifdef __x86_64__
1454 #if !HAS_BUILTIN(__lzcnt64)
1455 __INTRIN_INLINE unsigned long long __lzcnt64(unsigned long long value)
1456 {
1457  return __builtin_clzll(value);
1458 }
1459 #endif
1460 
1461 #if !HAS_BUILTIN(__popcnt64)
1462 __INTRIN_INLINE unsigned long long __popcnt64(unsigned long long value)
1463 {
1464  return __builtin_popcountll(value);
1465 }
1466 #endif
1467 #endif
1468 
1469 /*** 64-bit math ***/
1470 
1471 #if !HAS_BUILTIN(__emul)
1472 __INTRIN_INLINE long long __emul(int a, int b)
1473 {
1474  long long retval;
1475  __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1476  return retval;
1477 }
1478 #endif
1479 
1480 #if !HAS_BUILTIN(__emulu)
1481 __INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
1482 {
1483  unsigned long long retval;
1484  __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1485  return retval;
1486 }
1487 #endif
1488 
1489 __INTRIN_INLINE long long __cdecl _abs64(long long value)
1490 {
1491  return (value >= 0) ? value : -value;
1492 }
1493 
1494 #ifdef __x86_64__
1495 #if !HAS_BUILTIN(__mulh)
1496 __INTRIN_INLINE long long __mulh(long long a, long long b)
1497 {
1498  long long retval;
1499  __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1500  return retval;
1501 }
1502 #endif
1503 
1504 #if !HAS_BUILTIN(__umulh)
1505 __INTRIN_INLINE unsigned long long __umulh(unsigned long long a, unsigned long long b)
1506 {
1507  unsigned long long retval;
1508  __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1509  return retval;
1510 }
1511 #endif
1512 #endif
1513 
1514 /*** Port I/O ***/
1515 
1516 __INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
1517 {
1518  unsigned char byte;
1519  __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1520  return byte;
1521 }
1522 
1523 __INTRIN_INLINE unsigned short __inword(unsigned short Port)
1524 {
1525  unsigned short word;
1526  __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1527  return word;
1528 }
1529 
1530 __INTRIN_INLINE unsigned long __indword(unsigned short Port)
1531 {
1532  unsigned long dword;
1533  __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1534  return dword;
1535 }
1536 
1537 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1538 {
1539  __asm__ __volatile__
1540  (
1541  "rep; insb" :
1542  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1543  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1544  "memory"
1545  );
1546 }
1547 
1548 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1549 {
1550  __asm__ __volatile__
1551  (
1552  "rep; insw" :
1553  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1554  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1555  "memory"
1556  );
1557 }
1558 
1559 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1560 {
1561  __asm__ __volatile__
1562  (
1563  "rep; insl" :
1564  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1565  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1566  "memory"
1567  );
1568 }
1569 
1570 __INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
1571 {
1572  __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1573 }
1574 
1575 __INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
1576 {
1577  __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1578 }
1579 
1580 __INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
1581 {
1582  __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1583 }
1584 
1585 __INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1586 {
1587  __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1588 }
1589 
1590 __INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1591 {
1592  __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1593 }
1594 
1595 __INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1596 {
1597  __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1598 }
1599 
1600 __INTRIN_INLINE int __cdecl _inp(unsigned short Port)
1601 {
1602  return __inbyte(Port);
1603 }
1604 
1605 __INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
1606 {
1607  return __inword(Port);
1608 }
1609 
1610 __INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
1611 {
1612  return __indword(Port);
1613 }
1614 
1615 __INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
1616 {
1617  __outbyte(Port, (unsigned char)databyte);
1618  return databyte;
1619 }
1620 
1621 __INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
1622 {
1623  __outword(Port, dataword);
1624  return dataword;
1625 }
1626 
1627 __INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
1628 {
1629  __outdword(Port, dataword);
1630  return dataword;
1631 }
1632 
1633 
1634 /*** System information ***/
1635 
1636 __INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
1637 {
1638  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1639 }
1640 
1641 __INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
1642 {
1643  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType), "c" (ECXValue));
1644 }
1645 
1646 #if !HAS_BUILTIN(__rdtsc)
1647 __INTRIN_INLINE unsigned long long __rdtsc(void)
1648 {
1649 #ifdef __x86_64__
1650  unsigned long long low, high;
1651  __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1652  return low | (high << 32);
1653 #else
1654  unsigned long long retval;
1655  __asm__ __volatile__("rdtsc" : "=A"(retval));
1656  return retval;
1657 #endif
1658 }
1659 #endif /* !HAS_BUILTIN(__rdtsc) */
1660 
1662 {
1663  __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1664 }
1665 
1667 {
1668  uintptr_t retval;
1669  __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1670  return retval;
1671 }
1672 
1673 /*** Interrupts ***/
1674 
1675 #if !HAS_BUILTIN(__debugbreak)
1677 {
1678  __asm__("int $3");
1679 }
1680 #endif
1681 
1682 #if !HAS_BUILTIN(__ud2)
1684 {
1685  __asm__("ud2");
1686 }
1687 #endif
1688 
1689 #if !HAS_BUILTIN(__int2c)
1691 {
1692  __asm__("int $0x2c");
1693 }
1694 #endif
1695 
1697 {
1698  __asm__("cli" : : : "memory");
1699 }
1700 
1702 {
1703  __asm__("sti" : : : "memory");
1704 }
1705 
1707 {
1708  __asm__("hlt" : : : "memory");
1709 }
1710 
1711 #if !HAS_BUILTIN(__fastfail)
1712 __declspec(noreturn)
1713 __INTRIN_INLINE void __fastfail(unsigned int Code)
1714 {
1715  __asm__("int $0x29" : : "c"(Code) : "memory");
1716  __builtin_unreachable();
1717 }
1718 #endif
1719 
1720 /*** Protected memory management ***/
1721 
1722 #ifdef __x86_64__
1723 
1724 __INTRIN_INLINE void __writecr0(unsigned long long Data)
1725 {
1726  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1727 }
1728 
1729 __INTRIN_INLINE void __writecr3(unsigned long long Data)
1730 {
1731  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1732 }
1733 
1734 __INTRIN_INLINE void __writecr4(unsigned long long Data)
1735 {
1736  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1737 }
1738 
1739 __INTRIN_INLINE void __writecr8(unsigned long long Data)
1740 {
1741  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1742 }
1743 
1744 __INTRIN_INLINE unsigned long long __readcr0(void)
1745 {
1746  unsigned long long value;
1747  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1748  return value;
1749 }
1750 
1751 __INTRIN_INLINE unsigned long long __readcr2(void)
1752 {
1753  unsigned long long value;
1754  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1755  return value;
1756 }
1757 
1758 __INTRIN_INLINE unsigned long long __readcr3(void)
1759 {
1760  unsigned long long value;
1761  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1762  return value;
1763 }
1764 
1765 __INTRIN_INLINE unsigned long long __readcr4(void)
1766 {
1767  unsigned long long value;
1768  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1769  return value;
1770 }
1771 
1772 __INTRIN_INLINE unsigned long long __readcr8(void)
1773 {
1774  unsigned long long value;
1775  __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1776  return value;
1777 }
1778 
1779 #else /* __x86_64__ */
1780 
1781 __INTRIN_INLINE void __writecr0(unsigned int Data)
1782 {
1783  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1784 }
1785 
1786 __INTRIN_INLINE void __writecr3(unsigned int Data)
1787 {
1788  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1789 }
1790 
1791 __INTRIN_INLINE void __writecr4(unsigned int Data)
1792 {
1793  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1794 }
1795 
1796 __INTRIN_INLINE unsigned long __readcr0(void)
1797 {
1798  unsigned long value;
1799  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1800  return value;
1801 }
1802 
1803 __INTRIN_INLINE unsigned long __readcr2(void)
1804 {
1805  unsigned long value;
1806  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1807  return value;
1808 }
1809 
1810 __INTRIN_INLINE unsigned long __readcr3(void)
1811 {
1812  unsigned long value;
1813  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1814  return value;
1815 }
1816 
1817 __INTRIN_INLINE unsigned long __readcr4(void)
1818 {
1819  unsigned long value;
1820  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1821  return value;
1822 }
1823 
1824 #endif /* __x86_64__ */
1825 
1826 #ifdef __x86_64__
1827 
1828 __INTRIN_INLINE unsigned long long __readdr(unsigned int reg)
1829 {
1830  unsigned long long value;
1831  switch (reg)
1832  {
1833  case 0:
1834  __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1835  break;
1836  case 1:
1837  __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1838  break;
1839  case 2:
1840  __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1841  break;
1842  case 3:
1843  __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1844  break;
1845  case 4:
1846  __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1847  break;
1848  case 5:
1849  __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1850  break;
1851  case 6:
1852  __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1853  break;
1854  case 7:
1855  __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1856  break;
1857  }
1858  return value;
1859 }
1860 
1861 __INTRIN_INLINE void __writedr(unsigned reg, unsigned long long value)
1862 {
1863  switch (reg)
1864  {
1865  case 0:
1866  __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1867  break;
1868  case 1:
1869  __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1870  break;
1871  case 2:
1872  __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1873  break;
1874  case 3:
1875  __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1876  break;
1877  case 4:
1878  __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1879  break;
1880  case 5:
1881  __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1882  break;
1883  case 6:
1884  __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1885  break;
1886  case 7:
1887  __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1888  break;
1889  }
1890 }
1891 
1892 #else /* __x86_64__ */
1893 
1894 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1895 {
1896  unsigned int value;
1897  switch (reg)
1898  {
1899  case 0:
1900  __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1901  break;
1902  case 1:
1903  __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1904  break;
1905  case 2:
1906  __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1907  break;
1908  case 3:
1909  __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1910  break;
1911  case 4:
1912  __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1913  break;
1914  case 5:
1915  __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1916  break;
1917  case 6:
1918  __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1919  break;
1920  case 7:
1921  __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1922  break;
1923  }
1924  return value;
1925 }
1926 
1927 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1928 {
1929  switch (reg)
1930  {
1931  case 0:
1932  __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1933  break;
1934  case 1:
1935  __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1936  break;
1937  case 2:
1938  __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1939  break;
1940  case 3:
1941  __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1942  break;
1943  case 4:
1944  __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1945  break;
1946  case 5:
1947  __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1948  break;
1949  case 6:
1950  __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1951  break;
1952  case 7:
1953  __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1954  break;
1955  }
1956 }
1957 
1958 #endif /* __x86_64__ */
1959 
1961 {
1962  __asm__ __volatile__ ("invlpg (%[Address])" : : [Address] "b" (Address) : "memory");
1963 }
1964 
1965 
1966 /*** System operations ***/
1967 
1968 __INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
1969 {
1970 #ifdef __x86_64__
1971  unsigned long low, high;
1972  __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1973  return ((unsigned long long)high << 32) | low;
1974 #else
1975  unsigned long long retval;
1976  __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1977  return retval;
1978 #endif
1979 }
1980 
1981 __INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
1982 {
1983 #ifdef __x86_64__
1984  __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1985 #else
1986  __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1987 #endif
1988 }
1989 
1990 __INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
1991 {
1992  unsigned long long retval;
1993  __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
1994  return retval;
1995 }
1996 
1997 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1998 __INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
1999 {
2000  unsigned long retval;
2001  __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
2002  return retval;
2003 }
2004 
2006 {
2007  __asm__ __volatile__("wbinvd" : : : "memory");
2008 }
2009 
2011 {
2012  __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
2013 }
2014 
2016 {
2017  __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
2018 }
2019 
2021 {
2022  __asm__ __volatile__("sgdt %0" : : "m"(*(short*)Destination) : "memory");
2023 }
2024 
2025 /*** Misc operations ***/
2026 
2027 #if !HAS_BUILTIN(_mm_pause)
2029 {
2030  __asm__ __volatile__("pause" : : : "memory");
2031 }
2032 #endif
2033 
2035 {
2036  __asm__ __volatile__("nop");
2037 }
2038 
2039 #ifdef __cplusplus
2040 }
2041 #endif
2042 
2043 #endif /* KJK_INTRIN_X86_H_ */
2044 
2045 /* EOF */
__INTRIN_INLINE void __ud2(void)
Definition: intrin_x86.h:1683
CPPORT Port[4]
Definition: headless.c:35
__INTRIN_INLINE void __writecr4(unsigned int Data)
Definition: intrin_x86.h:1791
__INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
Definition: intrin_x86.h:1605
__INTRIN_INLINE void __stosd(unsigned long *Dest, unsigned long Data, size_t Count)
Definition: intrin_x86.h:782
__INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
Definition: intrin_x86.h:1927
Definition: pdh_main.c:93
__INTRIN_INLINE void _mm_lfence(void)
Definition: intrin_x86.h:106
__INTRIN_INLINE void __stosb(unsigned char *Dest, unsigned char Data, size_t Count)
Definition: intrin_x86.h:761
__INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
Definition: intrin_x86.h:1226
#define shift
Definition: input.c:1756
__INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:1008
__INTRIN_INLINE void *__cdecl memcpy(void *dest, const void *source, size_t num)
Definition: intrin_x86.h:74
__INTRIN_INLINE char _InterlockedCompareExchange8(volatile char *Destination, char Exchange, char Comperand)
Definition: intrin_x86.h:359
__INTRIN_INLINE unsigned long __readcr2(void)
Definition: intrin_x86.h:1803
__INTRIN_INLINE void __movsb(unsigned char *Destination, const unsigned char *Source, size_t Count)
Definition: intrin_x86.h:804
__INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
Definition: intrin_x86.h:1968
__INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long *a, long b)
Definition: intrin_x86.h:721
#define __cdecl
Definition: accygwin.h:79
__INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:950
__INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1200
__INTRIN_INLINE void __incfsdword(unsigned long Offset)
Definition: intrin_x86.h:1002
__INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
Definition: intrin_x86.h:984
__INTRIN_INLINE unsigned char _BitScanReverse(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1046
__INTRIN_INLINE void __wbinvd(void)
Definition: intrin_x86.h:2005
__INTRIN_INLINE long _InterlockedXor(volatile long *value, long mask)
Definition: intrin_x86.h:610
__INTRIN_INLINE void __incfsword(unsigned long Offset)
Definition: intrin_x86.h:997
__INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
Definition: intrin_x86.h:1288
__INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1537
__INTRIN_INLINE void __halt(void)
Definition: intrin_x86.h:1706
__INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long *Destination, long long Exchange, long long Comperand)
Definition: intrin_x86.h:683
__INTRIN_INLINE void __stosw(unsigned short *Dest, unsigned short Data, size_t Count)
Definition: intrin_x86.h:772
__INTRIN_INLINE void __lidt(void *Source)
Definition: intrin_x86.h:2010
__INTRIN_INLINE unsigned int __popcnt(unsigned int value)
Definition: intrin_x86.h:1440
GLint GLint GLint GLint GLint x
Definition: gl.h:1548
__INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1371
__INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
Definition: intrin_x86.h:1998
__declspec(noreturn) __INTRIN_INLINE void __fastfail(unsigned int Code)
Definition: intrin_x86.h:1712
__asm__("\n\t \ NewInt3Handler:\n\t \ pushl $" STR(REASON_INT3) "\n\t \ // call debugger loop\n\t \ jmp NewInt31Handler\n\t \ ")
__INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
Definition: intrin_x86.h:1408
__INTRIN_INLINE unsigned long __readcr3(void)
Definition: intrin_x86.h:1810
__INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
Definition: intrin_x86.h:966
#define __INTRIN_INLINE
Definition: _mingw.h:248
__INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1559
__INTRIN_INLINE void __movsd(unsigned long *Destination, const unsigned long *Source, size_t Count)
Definition: intrin_x86.h:824
__INTRIN_INLINE long long __emul(int a, int b)
Definition: intrin_x86.h:1472
IN OUT PLONG Addend
Definition: CrNtStubs.h:22
__INTRIN_INLINE short _InterlockedIncrement16(volatile short *lpAddend)
Definition: intrin_x86.h:650
__INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:955
__INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
Definition: intrin_x86.h:1209
__INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
Definition: intrin_x86.h:1270
__INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
Definition: intrin_x86.h:1426
__INTRIN_INLINE unsigned char _BitScanForward(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1038
__INTRIN_INLINE unsigned char _bittestandset(long *a, long b)
Definition: intrin_x86.h:1133
__INTRIN_INLINE long _InterlockedOr(volatile long *value, long mask)
Definition: intrin_x86.h:553
__INTRIN_INLINE int __cdecl _inp(unsigned short Port)
Definition: intrin_x86.h:1600
__INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
Definition: intrin_x86.h:1393
__INTRIN_INLINE void __cdecl __debugbreak(void)
Definition: intrin_x86.h:1676
__INTRIN_INLINE void __writecr3(unsigned int Data)
Definition: intrin_x86.h:1786
__INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1585
__INTRIN_INLINE unsigned char _bittestandreset(long *a, long b)
Definition: intrin_x86.h:1119
GLenum GLint GLuint mask
Definition: glext.h:6028
__INTRIN_INLINE uintptr_t __readeflags(void)
Definition: intrin_x86.h:1666
__INTRIN_INLINE long _InterlockedIncrement(volatile long *lpAddend)
Definition: intrin_x86.h:636
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:2015
__INTRIN_INLINE void _ReadWriteBarrier(void)
Definition: intrin_x86.h:88
#define _ReadBarrier
Definition: intrin_x86.h:95
__INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:1016
unsigned int uintptr_t
Definition: crtdefs.h:321
__INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:960
__INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long *const Addend, const long Value)
Definition: intrin_arm.h:359
__INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
Definition: intrin_x86.h:975
__INTRIN_INLINE void __writecr0(unsigned int Data)
Definition: intrin_x86.h:1781
static WCHAR Address[46]
Definition: ping.c:68
__INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
Definition: intrin_x86.h:1516
__INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile *Addend, char Value)
Definition: intrin_x86.h:431
__INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1244
__INTRIN_INLINE short _InterlockedDecrement16(volatile short *lpAddend)
Definition: intrin_x86.h:643
Definition: bufpool.h:45
__INTRIN_INLINE long _InterlockedCompareExchange(volatile long *Destination, long Exchange, long Comperand)
Definition: intrin_x86.h:377
_Must_inspect_result_ _In_ WDFKEY _In_ PCUNICODE_STRING _Out_opt_ PUSHORT _Inout_opt_ PUNICODE_STRING Value
Definition: wdfregistry.h:406
__INTRIN_INLINE void __nop(void)
Definition: intrin_x86.h:2034
__INTRIN_INLINE long _InterlockedAnd(volatile long *value, long mask)
Definition: intrin_x86.h:496
int Count
Definition: noreturn.cpp:7
__INTRIN_INLINE short _InterlockedAnd16(volatile short *value, short mask)
Definition: intrin_x86.h:477
__INTRIN_INLINE void _mm_pause(void)
Definition: intrin_x86.h:2028
GLboolean GLboolean GLboolean b
Definition: glext.h:6204
__INTRIN_INLINE long long __cdecl _abs64(long long value)
Definition: intrin_x86.h:1489
__INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
Definition: intrin_x86.h:1357
__INTRIN_INLINE char _InterlockedXor8(volatile char *value, char mask)
Definition: intrin_x86.h:572
__INTRIN_INLINE void * _InterlockedCompareExchangePointer(void *volatile *Destination, void *Exchange, void *Comperand)
Definition: intrin_x86.h:386
__INTRIN_INLINE long _InterlockedDecrement(volatile long *lpAddend)
Definition: intrin_x86.h:629
_In_ WDFCOLLECTION _In_ ULONG Index
__INTRIN_INLINE void * _InterlockedExchangePointer(void *volatile *Target, void *Value)
Definition: intrin_x86.h:422
__INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
Definition: intrin_x86.h:1235
__INTRIN_INLINE void __cdecl _enable(void)
Definition: intrin_x86.h:1701
__INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
Definition: intrin_x86.h:1981
GLuint GLuint num
Definition: glext.h:9618
__INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
Definition: intrin_x86.h:1636
_In_ PUNICODE_STRING _Inout_ PUNICODE_STRING Destination
Definition: rtlfuncs.h:2950
__INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
Definition: intrin_x86.h:1610
static int reg
Definition: i386-dis.c:1283
__INTRIN_INLINE char _InterlockedExchange8(volatile char *Target, char Value)
Definition: intrin_x86.h:395
__INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1595
__INTRIN_INLINE long _InterlockedExchangeAdd(volatile long *Addend, long Value)
Definition: intrin_x86.h:449
__INTRIN_INLINE void __int2c(void)
Definition: intrin_x86.h:1690
__INTRIN_INLINE long _InterlockedExchange(volatile long *Target, long Value)
Definition: intrin_x86.h:413
__INTRIN_INLINE short _InterlockedCompareExchange16(volatile short *Destination, short Exchange, short Comperand)
Definition: intrin_x86.h:368
__INTRIN_INLINE void __writeeflags(uintptr_t Value)
Definition: intrin_x86.h:1661
_In_ UCHAR _In_ UCHAR _In_ ULONG Code
Definition: wdfdevice.h:1697
__INTRIN_INLINE unsigned long long __rdtsc(void)
Definition: intrin_x86.h:1647
__INTRIN_INLINE void __cdecl _disable(void)
Definition: intrin_x86.h:1696
GLsizei const GLfloat * value
Definition: glext.h:6069
__INTRIN_INLINE unsigned int __readdr(unsigned int reg)
Definition: intrin_x86.h:1894
__INTRIN_INLINE char _InterlockedOr8(volatile char *value, char mask)
Definition: intrin_x86.h:515
__INTRIN_INLINE unsigned long __readcr0(void)
Definition: intrin_x86.h:1796
__INTRIN_INLINE unsigned long __readcr4(void)
Definition: intrin_x86.h:1817
processorSet Mask
__INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1548
__INTRIN_INLINE void _sgdt(void *Destination)
Definition: intrin_x86.h:2020
__INTRIN_INLINE unsigned long __indword(unsigned short Port)
Definition: intrin_x86.h:1530
__INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
Definition: intrin_x86.h:1481
__INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
Definition: intrin_x86.h:1279
#define local
Definition: zutil.h:30
_In_ ULONG _In_ ULONG Offset
Definition: ntddpcm.h:101
__INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
Definition: intrin_x86.h:1990
__INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
Definition: intrin_x86.h:1433
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1641
__INTRIN_INLINE char _InterlockedAnd8(volatile char *value, char mask)
Definition: intrin_x86.h:458
#define byte(x, n)
Definition: tomcrypt.h:118
__INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:1024
CPUINFO CPUInfo[]
Definition: parse.c:231
__INTRIN_INLINE void __incfsbyte(unsigned long Offset)
Definition: intrin_x86.h:992
__INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
Definition: intrin_x86.h:1621
__INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1343
void *__cdecl memmove(void *dest, const void *source, size_t num)
Definition: memmove.c:8
GLint GLint GLint GLint GLint GLint y
Definition: gl.h:1548
__INTRIN_INLINE short _InterlockedOr16(volatile short *value, short mask)
Definition: intrin_x86.h:534
const WCHAR * word
Definition: lex.c:36
_In_ WDFIOTARGET Target
Definition: wdfrequest.h:306
__INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
Definition: intrin_x86.h:1615
__INTRIN_INLINE short _InterlockedExchange16(volatile short *Target, short Value)
Definition: intrin_x86.h:404
__INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short *Addend, short Value)
Definition: intrin_x86.h:440
__INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1191
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1960
__INTRIN_INLINE unsigned char _bittest(const long *a, long b)
Definition: intrin_x86.h:1055
__INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1253
__INTRIN_INLINE short _InterlockedXor16(volatile short *value, short mask)
Definition: intrin_x86.h:591
__INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
Definition: intrin_x86.h:1386
__INTRIN_INLINE void _mm_mfence(void)
Definition: intrin_x86.h:99
__INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
Definition: intrin_x86.h:1627
_In_ UINT _In_ UINT _In_ PNDIS_PACKET Source
Definition: ndis.h:3167
static char * dest
Definition: rtl.c:135
__INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1590
GLboolean GLboolean GLboolean GLboolean a
Definition: glext.h:6204
__INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
Definition: intrin_x86.h:1575
__INTRIN_INLINE unsigned short __inword(unsigned short Port)
Definition: intrin_x86.h:1523
__INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
Definition: intrin_x86.h:1570
__INTRIN_INLINE void __movsw(unsigned short *Destination, const unsigned short *Source, size_t Count)
Definition: intrin_x86.h:814
__INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
Definition: intrin_x86.h:1580
__INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long *a, long b)
Definition: intrin_x86.h:740
__INTRIN_INLINE unsigned char _bittestandcomplement(long *a, long b)
Definition: intrin_x86.h:1105
__INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
Definition: intrin_x86.h:1447