ReactOS  0.4.15-dev-2965-g9a42267
intrin_x86.h
Go to the documentation of this file.
1 /*
2  Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3  Microsoft Visual C++ functions. Originally developed for the ReactOS
4  (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5  projects.
6 
7  Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8 
9  Permission is hereby granted, free of charge, to any person obtaining a
10  copy of this software and associated documentation files (the "Software"),
11  to deal in the Software without restriction, including without limitation
12  the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  and/or sell copies of the Software, and to permit persons to whom the
14  Software is furnished to do so, subject to the following conditions:
15 
16  The above copyright notice and this permission notice shall be included in
17  all copies or substantial portions of the Software.
18 
19  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25  DEALINGS IN THE SOFTWARE.
26 */
27 
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30 
31 /*
32  FIXME: review all "memory" clobbers, add/remove to match Visual C++
33  behavior: some "obvious" memory barriers are not present in the Visual C++
34  implementation - e.g. __stosX; on the other hand, some memory barriers that
35  *are* present could have been missed
36 */
37 
38 /*
39  NOTE: this is a *compatibility* header. Some functions may look wrong at
40  first, but they're only "as wrong" as they would be on Visual C++. Our
41  priority is compatibility
42 
43  NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44  constraints and the uses of __volatile__ out of my... hat. Do not touch
45  them. I hate cargo cult programming
46 
47  NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48  barriers aren't there in Visual C++ (e.g. __stosX)
49 
50  NOTE: review all intrinsics with a return value, add/remove __volatile__
51  where necessary. If an intrinsic whose value is ignored generates a no-op
52  under Visual C++, __volatile__ must be omitted; if it always generates code
53  (for example, if it has side effects), __volatile__ must be specified. GCC
54  will only optimize out non-volatile asm blocks with outputs, so input-only
55  blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56  and follow Visual C++ behavior
57 
58  NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59  atomic operations. Test the version like this:
60 
61  #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62  ...
63 
64  Pay attention to the type of barrier. Make it match with what Visual C++
65  would use in the same case
66 */
67 
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71 
72 /*** memcopy must be memmove ***/
73 void* __cdecl memmove(void* dest, const void* source, size_t num);
74 __INTRIN_INLINE void* __cdecl memcpy(void* dest, const void* source, size_t num)
75 {
76  return memmove(dest, source, num);
77 }
78 
79 
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
84 
85 /*** Memory barriers ***/
86 
87 #if !HAS_BUILTIN(_ReadWriteBarrier)
89 {
90  __asm__ __volatile__("" : : : "memory");
91 }
92 #endif
93 
94 /* GCC only supports full barriers */
95 #define _ReadBarrier _ReadWriteBarrier
96 #define _WriteBarrier _ReadWriteBarrier
97 
98 #if !HAS_BUILTIN(_mm_mfence)
100 {
101  __asm__ __volatile__("mfence" : : : "memory");
102 }
103 #endif
104 
105 #if !HAS_BUILTIN(_mm_lfence)
107 {
108  _ReadBarrier();
109  __asm__ __volatile__("lfence");
110  _ReadBarrier();
111 }
112 #endif
113 
114 #if !HAS_BUILTIN(_mm_sfence)
116 {
117  _WriteBarrier();
118  __asm__ __volatile__("sfence");
119  _WriteBarrier();
120 }
121 #endif
122 
123 #if defined(__x86_64__) && !HAS_BUILTIN(__faststorefence)
124 __INTRIN_INLINE void __faststorefence(void)
125 {
126  long local;
127  __asm__ __volatile__("lock; orl $0, %0;" : : "m"(local));
128 }
129 #endif
130 
131 
132 /*** Atomic operations ***/
133 
134 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
135 
136 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
137 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
138 {
139  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
140 }
141 #endif
142 
143 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
144 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
145 {
146  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
147 }
148 #endif
149 
150 #if !HAS_BUILTIN(_InterlockedCompareExchange)
151 __INTRIN_INLINE long __cdecl _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
152 {
153  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
154 }
155 #endif
156 
157 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
158 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
159 {
160  return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
161 }
162 #endif
163 
164 #if !HAS_BUILTIN(_InterlockedExchange8)
165 __INTRIN_INLINE char _InterlockedExchange8(volatile char * Target, char Value)
166 {
167  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
168  __sync_synchronize();
169  return __sync_lock_test_and_set(Target, Value);
170 }
171 #endif
172 
173 #if !HAS_BUILTIN(_InterlockedExchange16)
174 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
175 {
176  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
177  __sync_synchronize();
178  return __sync_lock_test_and_set(Target, Value);
179 }
180 #endif
181 
182 #if !HAS_BUILTIN(_InterlockedExchange)
183 __INTRIN_INLINE long __cdecl _InterlockedExchange(volatile long * Target, long Value)
184 {
185  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
186  __sync_synchronize();
187  return __sync_lock_test_and_set(Target, Value);
188 }
189 #endif
190 
191 #if !HAS_BUILTIN(_InterlockedExchangePointer)
192 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
193 {
194  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
195  __sync_synchronize();
196  return (void *)__sync_lock_test_and_set(Target, Value);
197 }
198 #endif
199 
200 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedExchange64)
201 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * Target, long long Value)
202 {
203  /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
204  __sync_synchronize();
205  return __sync_lock_test_and_set(Target, Value);
206 }
207 #endif
208 
209 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
210 __INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile * Addend, char Value)
211 {
212  return __sync_fetch_and_add(Addend, Value);
213 }
214 #endif
215 
216 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
217 __INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short * Addend, short Value)
218 {
219  return __sync_fetch_and_add(Addend, Value);
220 }
221 #endif
222 
223 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
224 __INTRIN_INLINE long __cdecl _InterlockedExchangeAdd(volatile long * Addend, long Value)
225 {
226  return __sync_fetch_and_add(Addend, Value);
227 }
228 #endif
229 
230 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedExchangeAdd64)
231 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * Addend, long long Value)
232 {
233  return __sync_fetch_and_add(Addend, Value);
234 }
235 #endif
236 
237 #if !HAS_BUILTIN(_InterlockedAnd8)
238 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
239 {
240  return __sync_fetch_and_and(value, mask);
241 }
242 #endif
243 
244 #if !HAS_BUILTIN(_InterlockedAnd16)
245 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
246 {
247  return __sync_fetch_and_and(value, mask);
248 }
249 #endif
250 
251 #if !HAS_BUILTIN(_InterlockedAnd)
252 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
253 {
254  return __sync_fetch_and_and(value, mask);
255 }
256 #endif
257 
258 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedAnd64)
259 __INTRIN_INLINE long long _InterlockedAnd64(volatile long long * value, long long mask)
260 {
261  return __sync_fetch_and_and(value, mask);
262 }
263 #endif
264 
265 #if !HAS_BUILTIN(_InterlockedOr8)
266 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
267 {
268  return __sync_fetch_and_or(value, mask);
269 }
270 #endif
271 
272 #if !HAS_BUILTIN(_InterlockedOr16)
273 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
274 {
275  return __sync_fetch_and_or(value, mask);
276 }
277 #endif
278 
279 #if !HAS_BUILTIN(_InterlockedOr)
280 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
281 {
282  return __sync_fetch_and_or(value, mask);
283 }
284 #endif
285 
286 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedOr64)
287 __INTRIN_INLINE long long _InterlockedOr64(volatile long long * value, long long mask)
288 {
289  return __sync_fetch_and_or(value, mask);
290 }
291 #endif
292 
293 #if !HAS_BUILTIN(_InterlockedXor8)
294 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
295 {
296  return __sync_fetch_and_xor(value, mask);
297 }
298 #endif
299 
300 #if !HAS_BUILTIN(_InterlockedXor16)
301 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
302 {
303  return __sync_fetch_and_xor(value, mask);
304 }
305 #endif
306 
307 #if !HAS_BUILTIN(_InterlockedXor)
308 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
309 {
310  return __sync_fetch_and_xor(value, mask);
311 }
312 #endif
313 
314 #if defined(__x86_64__) && !HAS_BUILTIN(_InterlockedXor64)
315 __INTRIN_INLINE long long _InterlockedXor64(volatile long long * value, long long mask)
316 {
317  return __sync_fetch_and_xor(value, mask);
318 }
319 #endif
320 
321 #if !HAS_BUILTIN(_InterlockedDecrement)
322 __INTRIN_INLINE long __cdecl _InterlockedDecrement(volatile long * lpAddend)
323 {
324  return __sync_sub_and_fetch(lpAddend, 1);
325 }
326 #endif
327 
328 #if !HAS_BUILTIN(_InterlockedIncrement)
329 __INTRIN_INLINE long __cdecl _InterlockedIncrement(volatile long * lpAddend)
330 {
331  return __sync_add_and_fetch(lpAddend, 1);
332 }
333 #endif
334 
335 #if !HAS_BUILTIN(_InterlockedDecrement16)
336 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
337 {
338  return __sync_sub_and_fetch(lpAddend, 1);
339 }
340 #endif
341 
342 #if !HAS_BUILTIN(_InterlockedIncrement16)
343 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
344 {
345  return __sync_add_and_fetch(lpAddend, 1);
346 }
347 #endif
348 
349 #if defined(__x86_64__)
350 #if !HAS_BUILTIN(_InterlockedDecrement64)
351 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
352 {
353  return __sync_sub_and_fetch(lpAddend, 1);
354 }
355 #endif
356 
357 #if !HAS_BUILTIN(_InterlockedIncrement64)
358 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
359 {
360  return __sync_add_and_fetch(lpAddend, 1);
361 }
362 #endif
363 #endif
364 
365 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
366 
367 #if !HAS_BUILTIN(_InterlockedCompareExchange8)
368 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * Destination, char Exchange, char Comperand)
369 {
370  char retval = Comperand;
371  __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
372  return retval;
373 }
374 #endif
375 
376 #if !HAS_BUILTIN(_InterlockedCompareExchange16)
377 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * Destination, short Exchange, short Comperand)
378 {
379  short retval = Comperand;
380  __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
381  return retval;
382 }
383 #endif
384 
385 #if !HAS_BUILTIN(_InterlockedCompareExchange)
386 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * Destination, long Exchange, long Comperand)
387 {
388  long retval = Comperand;
389  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
390  return retval;
391 }
392 #endif
393 
394 #if !HAS_BUILTIN(_InterlockedCompareExchangePointer)
395 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * Destination, void * Exchange, void * Comperand)
396 {
397  void * retval = (void *)Comperand;
398  __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
399  return retval;
400 }
401 #endif
402 
403 #if !HAS_BUILTIN(_InterlockedExchange8)
405 {
406  char retval = Value;
407  __asm__("xchgb %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
408  return retval;
409 }
410 #endif
411 
412 #if !HAS_BUILTIN(_InterlockedExchange16)
413 __INTRIN_INLINE short _InterlockedExchange16(volatile short * Target, short Value)
414 {
415  short retval = Value;
416  __asm__("xchgw %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
417  return retval;
418 }
419 #endif
420 
421 #if !HAS_BUILTIN(_InterlockedExchange)
423 {
424  long retval = Value;
425  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
426  return retval;
427 }
428 #endif
429 
430 #if !HAS_BUILTIN(_InterlockedExchangePointer)
431 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * Target, void * Value)
432 {
433  void * retval = Value;
434  __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
435  return retval;
436 }
437 #endif
438 
439 #if !HAS_BUILTIN(_InterlockedExchangeAdd8)
441 {
442  char retval = Value;
443  __asm__("lock; xaddb %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
444  return retval;
445 }
446 #endif
447 
448 #if !HAS_BUILTIN(_InterlockedExchangeAdd16)
450 {
451  short retval = Value;
452  __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
453  return retval;
454 }
455 #endif
456 
457 #if !HAS_BUILTIN(_InterlockedExchangeAdd)
459 {
460  long retval = Value;
461  __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
462  return retval;
463 }
464 #endif
465 
466 #if !HAS_BUILTIN(_InterlockedAnd8)
467 __INTRIN_INLINE char _InterlockedAnd8(volatile char * value, char mask)
468 {
469  char x;
470  char y;
471 
472  y = *value;
473 
474  do
475  {
476  x = y;
478  }
479  while(y != x);
480 
481  return y;
482 }
483 #endif
484 
485 #if !HAS_BUILTIN(_InterlockedAnd16)
486 __INTRIN_INLINE short _InterlockedAnd16(volatile short * value, short mask)
487 {
488  short x;
489  short y;
490 
491  y = *value;
492 
493  do
494  {
495  x = y;
497  }
498  while(y != x);
499 
500  return y;
501 }
502 #endif
503 
504 #if !HAS_BUILTIN(_InterlockedAnd)
505 __INTRIN_INLINE long _InterlockedAnd(volatile long * value, long mask)
506 {
507  long x;
508  long y;
509 
510  y = *value;
511 
512  do
513  {
514  x = y;
516  }
517  while(y != x);
518 
519  return y;
520 }
521 #endif
522 
523 #if !HAS_BUILTIN(_InterlockedOr8)
524 __INTRIN_INLINE char _InterlockedOr8(volatile char * value, char mask)
525 {
526  char x;
527  char y;
528 
529  y = *value;
530 
531  do
532  {
533  x = y;
535  }
536  while(y != x);
537 
538  return y;
539 }
540 #endif
541 
542 #if !HAS_BUILTIN(_InterlockedOr16)
543 __INTRIN_INLINE short _InterlockedOr16(volatile short * value, short mask)
544 {
545  short x;
546  short y;
547 
548  y = *value;
549 
550  do
551  {
552  x = y;
554  }
555  while(y != x);
556 
557  return y;
558 }
559 #endif
560 
561 #if !HAS_BUILTIN(_InterlockedOr)
562 __INTRIN_INLINE long _InterlockedOr(volatile long * value, long mask)
563 {
564  long x;
565  long y;
566 
567  y = *value;
568 
569  do
570  {
571  x = y;
573  }
574  while(y != x);
575 
576  return y;
577 }
578 #endif
579 
580 #if !HAS_BUILTIN(_InterlockedXor8)
581 __INTRIN_INLINE char _InterlockedXor8(volatile char * value, char mask)
582 {
583  char x;
584  char y;
585 
586  y = *value;
587 
588  do
589  {
590  x = y;
592  }
593  while(y != x);
594 
595  return y;
596 }
597 #endif
598 
599 #if !HAS_BUILTIN(_InterlockedXor16)
600 __INTRIN_INLINE short _InterlockedXor16(volatile short * value, short mask)
601 {
602  short x;
603  short y;
604 
605  y = *value;
606 
607  do
608  {
609  x = y;
611  }
612  while(y != x);
613 
614  return y;
615 }
616 #endif
617 
618 #if !HAS_BUILTIN(_InterlockedXor)
619 __INTRIN_INLINE long _InterlockedXor(volatile long * value, long mask)
620 {
621  long x;
622  long y;
623 
624  y = *value;
625 
626  do
627  {
628  x = y;
630  }
631  while(y != x);
632 
633  return y;
634 }
635 #endif
636 
637 #if !HAS_BUILTIN(_InterlockedDecrement)
638 __INTRIN_INLINE long _InterlockedDecrement(volatile long * lpAddend)
639 {
640  return _InterlockedExchangeAdd(lpAddend, -1) - 1;
641 }
642 #endif
643 
644 #if !HAS_BUILTIN(_InterlockedIncrement)
645 __INTRIN_INLINE long _InterlockedIncrement(volatile long * lpAddend)
646 {
647  return _InterlockedExchangeAdd(lpAddend, 1) + 1;
648 }
649 #endif
650 
651 #if !HAS_BUILTIN(_InterlockedDecrement16)
652 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * lpAddend)
653 {
654  return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
655 }
656 #endif
657 
658 #if !HAS_BUILTIN(_InterlockedIncrement16)
659 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * lpAddend)
660 {
661  return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
662 }
663 #endif
664 
665 #if defined(__x86_64__)
666 #if !HAS_BUILTIN(_InterlockedDecrement64)
667 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * lpAddend)
668 {
669  return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
670 }
671 #endif
672 
673 #if !HAS_BUILTIN(_InterlockedIncrement64)
674 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * lpAddend)
675 {
676  return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
677 }
678 #endif
679 #endif
680 
681 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
682 
683 #if !HAS_BUILTIN(_InterlockedCompareExchange64)
684 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
685 
686 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
687 {
688  return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
689 }
690 
691 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
692 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * Destination, long long Exchange, long long Comperand)
693 {
694  long long retval = Comperand;
695 
696  __asm__
697  (
698  "lock; cmpxchg8b %[Destination]" :
699  [retval] "+A" (retval) :
700  [Destination] "m" (*Destination),
701  "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
702  "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
703  "memory"
704  );
705 
706  return retval;
707 }
708 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__) */
709 #endif /* !HAS_BUILTIN(_InterlockedCompareExchange64) */
710 
711 #ifdef __i386__
712 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * Addend, long Value)
713 {
714  __asm__
715  (
716  "lock; addl %[Value], %[Lo32];"
717  "jae LABEL%=;"
718  "lock; adcl $0, %[Hi32];"
719  "LABEL%=:;" :
720  [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
721  [Value] "ir" (Value) :
722  "memory"
723  );
724 
725  return Value;
726 }
727 #endif /* __i386__ */
728 
729 #if !HAS_BUILTIN(_interlockedbittestandreset)
730 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, long b)
731 {
732  unsigned char retval;
733  __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
734  return retval;
735 }
736 #endif
737 
738 #if defined(__x86_64__) && !HAS_BUILTIN(_interlockedbittestandreset64)
739 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, long long b)
740 {
741  unsigned char retval;
742  __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
743  return retval;
744 }
745 
746 #endif
747 
748 #if !HAS_BUILTIN(_interlockedbittestandset)
749 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, long b)
750 {
751  unsigned char retval;
752  __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
753  return retval;
754 }
755 #endif
756 
757 #if defined(__x86_64__) && !HAS_BUILTIN(_interlockedbittestandset64)
758 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, long long b)
759 {
760  unsigned char retval;
761  __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
762  return retval;
763 }
764 #endif
765 
766 /*** String operations ***/
767 
768 #if !HAS_BUILTIN(__stosb)
769 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
770 __INTRIN_INLINE void __stosb(unsigned char * Dest, unsigned char Data, size_t Count)
771 {
772  __asm__ __volatile__
773  (
774  "rep; stosb" :
775  [Dest] "=D" (Dest), [Count] "=c" (Count) :
776  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
777  );
778 }
779 #endif
780 
781 __INTRIN_INLINE void __stosw(unsigned short * Dest, unsigned short Data, size_t Count)
782 {
783  __asm__ __volatile__
784  (
785  "rep; stosw" :
786  [Dest] "=D" (Dest), [Count] "=c" (Count) :
787  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
788  );
789 }
790 
791 __INTRIN_INLINE void __stosd(unsigned long * Dest, unsigned long Data, size_t Count)
792 {
793  __asm__ __volatile__
794  (
795  "rep; stosl" :
796  [Dest] "=D" (Dest), [Count] "=c" (Count) :
797  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
798  );
799 }
800 
801 #ifdef __x86_64__
802 __INTRIN_INLINE void __stosq(unsigned long long * Dest, unsigned long long Data, size_t Count)
803 {
804  __asm__ __volatile__
805  (
806  "rep; stosq" :
807  [Dest] "=D" (Dest), [Count] "=c" (Count) :
808  "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
809  );
810 }
811 #endif
812 
813 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
814 {
815  __asm__ __volatile__
816  (
817  "rep; movsb" :
818  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
819  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
820  );
821 }
822 
823 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
824 {
825  __asm__ __volatile__
826  (
827  "rep; movsw" :
828  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
829  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
830  );
831 }
832 
833 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
834 {
835  __asm__ __volatile__
836  (
837  "rep; movsd" :
838  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
839  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
840  );
841 }
842 
843 #ifdef __x86_64__
844 __INTRIN_INLINE void __movsq(unsigned long long * Destination, const unsigned long long * Source, size_t Count)
845 {
846  __asm__ __volatile__
847  (
848  "rep; movsq" :
849  [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
850  "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
851  );
852 }
853 #endif
854 
855 #if defined(__x86_64__)
856 
857 /*** GS segment addressing ***/
858 
859 __INTRIN_INLINE void __writegsbyte(unsigned long Offset, unsigned char Data)
860 {
861  __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
862 }
863 
864 __INTRIN_INLINE void __writegsword(unsigned long Offset, unsigned short Data)
865 {
866  __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
867 }
868 
869 __INTRIN_INLINE void __writegsdword(unsigned long Offset, unsigned long Data)
870 {
871  __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
872 }
873 
874 __INTRIN_INLINE void __writegsqword(unsigned long Offset, unsigned long long Data)
875 {
876  __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
877 }
878 
879 #if !HAS_BUILTIN(__readgsbyte)
880 __INTRIN_INLINE unsigned char __readgsbyte(unsigned long Offset)
881 {
882  unsigned char value;
883  __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
884  return value;
885 }
886 #endif
887 
888 #if !HAS_BUILTIN(__readgsword)
889 __INTRIN_INLINE unsigned short __readgsword(unsigned long Offset)
890 {
891  unsigned short value;
892  __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
893  return value;
894 }
895 #endif
896 
897 #if !HAS_BUILTIN(__readgsdword)
898 __INTRIN_INLINE unsigned long __readgsdword(unsigned long Offset)
899 {
900  unsigned long value;
901  __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
902  return value;
903 }
904 #endif
905 
906 #if !HAS_BUILTIN(__readgsqword)
907 __INTRIN_INLINE unsigned long long __readgsqword(unsigned long Offset)
908 {
909  unsigned long long value;
910  __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
911  return value;
912 }
913 #endif
914 
915 __INTRIN_INLINE void __incgsbyte(unsigned long Offset)
916 {
917  __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
918 }
919 
920 __INTRIN_INLINE void __incgsword(unsigned long Offset)
921 {
922  __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
923 }
924 
925 __INTRIN_INLINE void __incgsdword(unsigned long Offset)
926 {
927  __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
928 }
929 
930 __INTRIN_INLINE void __incgsqword(unsigned long Offset)
931 {
932  __asm__ __volatile__("incq %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
933 }
934 
935 __INTRIN_INLINE void __addgsbyte(unsigned long Offset, unsigned char Data)
936 {
937  __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
938 }
939 
940 __INTRIN_INLINE void __addgsword(unsigned long Offset, unsigned short Data)
941 {
942  __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
943 }
944 
945 __INTRIN_INLINE void __addgsdword(unsigned long Offset, unsigned long Data)
946 {
947  __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
948 }
949 
950 __INTRIN_INLINE void __addgsqword(unsigned long Offset, unsigned long long Data)
951 {
952  __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
953 }
954 
955 #else /* defined(__x86_64__) */
956 
957 /*** FS segment addressing ***/
958 
959 __INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
960 {
961  __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
962 }
963 
964 __INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
965 {
966  __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
967 }
968 
969 __INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
970 {
971  __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
972 }
973 
974 #if !HAS_BUILTIN(__readfsbyte)
975 __INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
976 {
977  unsigned char value;
978  __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
979  return value;
980 }
981 #endif
982 
983 #if !HAS_BUILTIN(__readfsword)
984 __INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
985 {
986  unsigned short value;
987  __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
988  return value;
989 }
990 #endif
991 
992 #if !HAS_BUILTIN(__readfsdword)
993 __INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
994 {
995  unsigned long value;
996  __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
997  return value;
998 }
999 #endif
1000 
1002 {
1003  __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
1004 }
1005 
1007 {
1008  __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
1009 }
1010 
1012 {
1013  __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
1014 }
1015 
1016 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
1017 __INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
1018 {
1019  if(!__builtin_constant_p(Offset))
1020  __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1021  else
1022  __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1023 }
1024 
1025 __INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
1026 {
1027  if(!__builtin_constant_p(Offset))
1028  __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1029  else
1030  __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1031 }
1032 
1033 __INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
1034 {
1035  if(!__builtin_constant_p(Offset))
1036  __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
1037  else
1038  __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
1039 }
1040 
1041 #endif /* defined(__x86_64__) */
1042 
1043 
1044 /*** Bit manipulation ***/
1045 
1046 #if !HAS_BUILTIN(_BitScanForward)
1047 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * Index, unsigned long Mask)
1048 {
1049  __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1050  return Mask ? 1 : 0;
1051 }
1052 #endif
1053 
1054 #if !HAS_BUILTIN(_BitScanReverse)
1055 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * Index, unsigned long Mask)
1056 {
1057  __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
1058  return Mask ? 1 : 0;
1059 }
1060 #endif
1061 
1062 #if !HAS_BUILTIN(_bittest)
1063 /* NOTE: again, the bizarre implementation follows Visual C++ */
1064 __INTRIN_INLINE unsigned char _bittest(const long * a, long b)
1065 {
1066  unsigned char retval;
1067 
1068  if(__builtin_constant_p(b))
1069  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
1070  else
1071  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1072 
1073  return retval;
1074 }
1075 #endif
1076 
1077 #ifdef __x86_64__
1078 #if !HAS_BUILTIN(_BitScanForward64)
1079 __INTRIN_INLINE unsigned char _BitScanForward64(unsigned long * Index, unsigned long long Mask)
1080 {
1081  unsigned long long Index64;
1082  __asm__("bsfq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1083  *Index = Index64;
1084  return Mask ? 1 : 0;
1085 }
1086 #endif
1087 
1088 #if !HAS_BUILTIN(_BitScanReverse64)
1089 __INTRIN_INLINE unsigned char _BitScanReverse64(unsigned long * Index, unsigned long long Mask)
1090 {
1091  unsigned long long Index64;
1092  __asm__("bsrq %[Mask], %[Index]" : [Index] "=r" (Index64) : [Mask] "mr" (Mask));
1093  *Index = Index64;
1094  return Mask ? 1 : 0;
1095 }
1096 #endif
1097 
1098 #if !HAS_BUILTIN(_bittest64)
1099 __INTRIN_INLINE unsigned char _bittest64(const long long * a, long long b)
1100 {
1101  unsigned char retval;
1102 
1103  if(__builtin_constant_p(b))
1104  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
1105  else
1106  __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "m" (*a), [b] "r" (b));
1107 
1108  return retval;
1109 }
1110 #endif
1111 #endif
1112 
1113 #if !HAS_BUILTIN(_bittestandcomplement)
1114 __INTRIN_INLINE unsigned char _bittestandcomplement(long * a, long b)
1115 {
1116  unsigned char retval;
1117 
1118  if(__builtin_constant_p(b))
1119  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1120  else
1121  __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1122 
1123  return retval;
1124 }
1125 #endif
1126 
1127 #if !HAS_BUILTIN(_bittestandreset)
1128 __INTRIN_INLINE unsigned char _bittestandreset(long * a, long b)
1129 {
1130  unsigned char retval;
1131 
1132  if(__builtin_constant_p(b))
1133  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1134  else
1135  __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1136 
1137  return retval;
1138 }
1139 #endif
1140 
1141 #if !HAS_BUILTIN(_bittestandset)
1142 __INTRIN_INLINE unsigned char _bittestandset(long * a, long b)
1143 {
1144  unsigned char retval;
1145 
1146  if(__builtin_constant_p(b))
1147  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
1148  else
1149  __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1150 
1151  return retval;
1152 }
1153 #endif
1154 
1155 #ifdef __x86_64__
1156 #if !HAS_BUILTIN(_bittestandset64)
1157 __INTRIN_INLINE unsigned char _bittestandset64(long long * a, long long b)
1158 {
1159  unsigned char retval;
1160 
1161  if(__builtin_constant_p(b))
1162  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1163  else
1164  __asm__("btsq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1165 
1166  return retval;
1167 }
1168 #endif
1169 
1170 #if !HAS_BUILTIN(_bittestandreset64)
1171 __INTRIN_INLINE unsigned char _bittestandreset64(long long * a, long long b)
1172 {
1173  unsigned char retval;
1174 
1175  if(__builtin_constant_p(b))
1176  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1177  else
1178  __asm__("btrq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1179 
1180  return retval;
1181 }
1182 #endif
1183 
1184 #if !HAS_BUILTIN(_bittestandcomplement64)
1185 __INTRIN_INLINE unsigned char _bittestandcomplement64(long long * a, long long b)
1186 {
1187  unsigned char retval;
1188 
1189  if(__builtin_constant_p(b))
1190  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 64))), [retval] "=q" (retval) : [b] "Ir" (b % 64));
1191  else
1192  __asm__("btcq %[b], %[a]; setb %b[retval]" : [a] "+m" (*a), [retval] "=q" (retval) : [b] "r" (b));
1193 
1194  return retval;
1195 }
1196 #endif
1197 #endif /* __x86_64__ */
1198 
1199 #if !HAS_BUILTIN(_rotl8)
1200 __INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
1201 {
1202  unsigned char retval;
1203  __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1204  return retval;
1205 }
1206 #endif
1207 
1208 #if !HAS_BUILTIN(_rotl16)
1209 __INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
1210 {
1211  unsigned short retval;
1212  __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1213  return retval;
1214 }
1215 #endif
1216 
1217 #if !HAS_BUILTIN(_rotl)
1218 __INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
1219 {
1220  unsigned int retval;
1221  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1222  return retval;
1223 }
1224 #endif
1225 
1226 #if !HAS_BUILTIN(_rotl64)
1227 #ifdef __x86_64__
1228 __INTRIN_INLINE unsigned long long _rotl64(unsigned long long value, int shift)
1229 {
1230  unsigned long long retval;
1231  __asm__("rolq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1232  return retval;
1233 }
1234 #else /* __x86_64__ */
1235 __INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
1236 {
1237  /* FIXME: this is probably not optimal */
1238  return (value << shift) | (value >> (64 - shift));
1239 }
1240 #endif /* __x86_64__ */
1241 #endif /* !HAS_BUILTIN(_rotl64) */
1242 
1243 #if !HAS_BUILTIN(_rotr)
1244 __INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
1245 {
1246  unsigned int retval;
1247  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1248  return retval;
1249 }
1250 #endif
1251 
1252 #if !HAS_BUILTIN(_rotr8)
1253 __INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
1254 {
1255  unsigned char retval;
1256  __asm__("rorb %b[shift], %b[retval]" : [retval] "=qm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1257  return retval;
1258 }
1259 #endif
1260 
1261 #if !HAS_BUILTIN(_rotr16)
1262 __INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
1263 {
1264  unsigned short retval;
1265  __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1266  return retval;
1267 }
1268 #endif
1269 
1270 #if !HAS_BUILTIN(_rotr64)
1271 #ifdef __x86_64__
1272 __INTRIN_INLINE unsigned long long _rotr64(unsigned long long value, int shift)
1273 {
1274  unsigned long long retval;
1275  __asm__("rorq %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1276  return retval;
1277 }
1278 #else /* __x86_64__ */
1279 __INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
1280 {
1281  /* FIXME: this is probably not optimal */
1282  return (value >> shift) | (value << (64 - shift));
1283 }
1284 #endif /* __x86_64__ */
1285 #endif /* !HAS_BUILTIN(_rotr64) */
1286 
1287 #if !HAS_BUILTIN(_lrotl)
1288 __INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
1289 {
1290  unsigned long retval;
1291  __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1292  return retval;
1293 }
1294 #endif
1295 
1296 #if !HAS_BUILTIN(_lrotr)
1297 __INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
1298 {
1299  unsigned long retval;
1300  __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
1301  return retval;
1302 }
1303 #endif
1304 
1305 #ifdef __x86_64__
1306 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1307 {
1308  unsigned long long retval;
1309  unsigned char shift = Bit & 0x3F;
1310 
1311  __asm__
1312  (
1313  "shlq %[shift], %[Mask]" : "=r"(retval) : [Mask] "0"(Mask), [shift] "c"(shift)
1314  );
1315 
1316  return retval;
1317 }
1318 
1319 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1320 {
1321  long long retval;
1322  unsigned char shift = Bit & 0x3F;
1323 
1324  __asm__
1325  (
1326  "sarq %[shift], %[Mask]" : "=r"(retval) : [Mask] "0"(Mask), [shift] "c"(shift)
1327  );
1328 
1329  return retval;
1330 }
1331 
1332 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1333 {
1334  long long retval;
1335  unsigned char shift = Bit & 0x3F;
1336 
1337  __asm__
1338  (
1339  "shrq %[shift], %[Mask]" : "=r"(retval) : [Mask] "0"(Mask), [shift] "c"(shift)
1340  );
1341 
1342  return retval;
1343 }
1344 #else
1345 /*
1346  NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1347  constraint (edx:eax) for the Mask argument, because it's the only way GCC
1348  can pass 64-bit operands around - passing the two 32 bit parts separately
1349  just confuses it. Also we declare Bit as an int and then truncate it to
1350  match Visual C++ behavior
1351 */
1352 __INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
1353 {
1354  unsigned long long retval = Mask;
1355 
1356  __asm__
1357  (
1358  "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1359  "+A" (retval) :
1360  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1361  );
1362 
1363  return retval;
1364 }
1365 
1366 __INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
1367 {
1368  long long retval = Mask;
1369 
1370  __asm__
1371  (
1372  "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1373  "+A" (retval) :
1374  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1375  );
1376 
1377  return retval;
1378 }
1379 
1380 __INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
1381 {
1382  unsigned long long retval = Mask;
1383 
1384  __asm__
1385  (
1386  "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1387  "+A" (retval) :
1388  [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
1389  );
1390 
1391  return retval;
1392 }
1393 #endif
1394 
1395 __INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
1396 {
1397  unsigned short retval;
1398  __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
1399  return retval;
1400 }
1401 
1402 __INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
1403 {
1404  unsigned long retval;
1405  __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1406  return retval;
1407 }
1408 
1409 #ifdef __x86_64__
1410 __INTRIN_INLINE unsigned long long _byteswap_uint64(unsigned long long value)
1411 {
1412  unsigned long long retval;
1413  __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
1414  return retval;
1415 }
1416 #else
1417 __INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
1418 {
1419  union {
1420  unsigned long long int64part;
1421  struct {
1422  unsigned long lowpart;
1423  unsigned long hipart;
1424  };
1425  } retval;
1426  retval.int64part = value;
1427  __asm__("bswapl %[lowpart]\n"
1428  "bswapl %[hipart]\n"
1429  : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
1430  return retval.int64part;
1431 }
1432 #endif
1433 
1434 #if !HAS_BUILTIN(__lzcnt)
1435 __INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
1436 {
1437  return __builtin_clz(value);
1438 }
1439 #endif
1440 
1441 #if !HAS_BUILTIN(__lzcnt16)
1442 __INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
1443 {
1444  return __builtin_clz(value);
1445 }
1446 #endif
1447 
1448 #if !HAS_BUILTIN(__popcnt)
1449 __INTRIN_INLINE unsigned int __popcnt(unsigned int value)
1450 {
1451  return __builtin_popcount(value);
1452 }
1453 #endif
1454 
1455 #if !HAS_BUILTIN(__popcnt16)
1456 __INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
1457 {
1458  return __builtin_popcount(value);
1459 }
1460 #endif
1461 
1462 #ifdef __x86_64__
1463 #if !HAS_BUILTIN(__lzcnt64)
1464 __INTRIN_INLINE unsigned long long __lzcnt64(unsigned long long value)
1465 {
1466  return __builtin_clzll(value);
1467 }
1468 #endif
1469 
1470 #if !HAS_BUILTIN(__popcnt64)
1471 __INTRIN_INLINE unsigned long long __popcnt64(unsigned long long value)
1472 {
1473  return __builtin_popcountll(value);
1474 }
1475 #endif
1476 #endif
1477 
1478 /*** 64-bit math ***/
1479 
1480 #if !HAS_BUILTIN(__emul)
1481 __INTRIN_INLINE long long __emul(int a, int b)
1482 {
1483  long long retval;
1484  __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1485  return retval;
1486 }
1487 #endif
1488 
1489 #if !HAS_BUILTIN(__emulu)
1490 __INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
1491 {
1492  unsigned long long retval;
1493  __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1494  return retval;
1495 }
1496 #endif
1497 
1498 __INTRIN_INLINE long long __cdecl _abs64(long long value)
1499 {
1500  return (value >= 0) ? value : -value;
1501 }
1502 
1503 #ifdef __x86_64__
1504 #if !HAS_BUILTIN(__mulh)
1505 __INTRIN_INLINE long long __mulh(long long a, long long b)
1506 {
1507  long long retval;
1508  __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1509  return retval;
1510 }
1511 #endif
1512 
1513 #if !HAS_BUILTIN(__umulh)
1514 __INTRIN_INLINE unsigned long long __umulh(unsigned long long a, unsigned long long b)
1515 {
1516  unsigned long long retval;
1517  __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1518  return retval;
1519 }
1520 #endif
1521 #endif
1522 
1523 /*** Port I/O ***/
1524 
1525 __INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
1526 {
1527  unsigned char byte;
1528  __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1529  return byte;
1530 }
1531 
1532 __INTRIN_INLINE unsigned short __inword(unsigned short Port)
1533 {
1534  unsigned short word;
1535  __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1536  return word;
1537 }
1538 
1539 __INTRIN_INLINE unsigned long __indword(unsigned short Port)
1540 {
1541  unsigned long dword;
1542  __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1543  return dword;
1544 }
1545 
1546 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1547 {
1548  __asm__ __volatile__
1549  (
1550  "rep; insb" :
1551  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1552  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1553  "memory"
1554  );
1555 }
1556 
1557 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1558 {
1559  __asm__ __volatile__
1560  (
1561  "rep; insw" :
1562  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1563  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1564  "memory"
1565  );
1566 }
1567 
1568 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1569 {
1570  __asm__ __volatile__
1571  (
1572  "rep; insl" :
1573  [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1574  "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1575  "memory"
1576  );
1577 }
1578 
1579 __INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
1580 {
1581  __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1582 }
1583 
1584 __INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
1585 {
1586  __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1587 }
1588 
1589 __INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
1590 {
1591  __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1592 }
1593 
1594 __INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1595 {
1596  __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1597 }
1598 
1599 __INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1600 {
1601  __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1602 }
1603 
1604 __INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1605 {
1606  __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1607 }
1608 
1609 __INTRIN_INLINE int __cdecl _inp(unsigned short Port)
1610 {
1611  return __inbyte(Port);
1612 }
1613 
1614 __INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
1615 {
1616  return __inword(Port);
1617 }
1618 
1619 __INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
1620 {
1621  return __indword(Port);
1622 }
1623 
1624 __INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
1625 {
1626  __outbyte(Port, (unsigned char)databyte);
1627  return databyte;
1628 }
1629 
1630 __INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
1631 {
1632  __outword(Port, dataword);
1633  return dataword;
1634 }
1635 
1636 __INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
1637 {
1638  __outdword(Port, dataword);
1639  return dataword;
1640 }
1641 
1642 
1643 /*** System information ***/
1644 
1645 __INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
1646 {
1647  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1648 }
1649 
1650 __INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
1651 {
1652  __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType), "c" (ECXValue));
1653 }
1654 
1655 #if !HAS_BUILTIN(__rdtsc)
1656 __INTRIN_INLINE unsigned long long __rdtsc(void)
1657 {
1658 #ifdef __x86_64__
1659  unsigned long long low, high;
1660  __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1661  return low | (high << 32);
1662 #else
1663  unsigned long long retval;
1664  __asm__ __volatile__("rdtsc" : "=A"(retval));
1665  return retval;
1666 #endif
1667 }
1668 #endif /* !HAS_BUILTIN(__rdtsc) */
1669 
1671 {
1672  __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1673 }
1674 
1676 {
1677  uintptr_t retval;
1678  __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1679  return retval;
1680 }
1681 
1682 /*** Interrupts ***/
1683 
1684 #if !HAS_BUILTIN(__debugbreak)
1686 {
1687  __asm__("int $3");
1688 }
1689 #endif
1690 
1691 #if !HAS_BUILTIN(__ud2)
1693 {
1694  __asm__("ud2");
1695 }
1696 #endif
1697 
1698 #if !HAS_BUILTIN(__int2c)
1700 {
1701  __asm__("int $0x2c");
1702 }
1703 #endif
1704 
1706 {
1707  __asm__("cli" : : : "memory");
1708 }
1709 
1711 {
1712  __asm__("sti" : : : "memory");
1713 }
1714 
1716 {
1717  __asm__("hlt" : : : "memory");
1718 }
1719 
1720 #if !HAS_BUILTIN(__fastfail)
1721 __declspec(noreturn)
1722 __INTRIN_INLINE void __fastfail(unsigned int Code)
1723 {
1724  __asm__("int $0x29" : : "c"(Code) : "memory");
1725  __builtin_unreachable();
1726 }
1727 #endif
1728 
1729 /*** Protected memory management ***/
1730 
1731 #ifdef __x86_64__
1732 
1733 __INTRIN_INLINE void __writecr0(unsigned long long Data)
1734 {
1735  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1736 }
1737 
1738 __INTRIN_INLINE void __writecr3(unsigned long long Data)
1739 {
1740  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1741 }
1742 
1743 __INTRIN_INLINE void __writecr4(unsigned long long Data)
1744 {
1745  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1746 }
1747 
1748 __INTRIN_INLINE void __writecr8(unsigned long long Data)
1749 {
1750  __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1751 }
1752 
1753 __INTRIN_INLINE unsigned long long __readcr0(void)
1754 {
1755  unsigned long long value;
1756  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1757  return value;
1758 }
1759 
1760 __INTRIN_INLINE unsigned long long __readcr2(void)
1761 {
1762  unsigned long long value;
1763  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1764  return value;
1765 }
1766 
1767 __INTRIN_INLINE unsigned long long __readcr3(void)
1768 {
1769  unsigned long long value;
1770  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1771  return value;
1772 }
1773 
1774 __INTRIN_INLINE unsigned long long __readcr4(void)
1775 {
1776  unsigned long long value;
1777  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1778  return value;
1779 }
1780 
1781 __INTRIN_INLINE unsigned long long __readcr8(void)
1782 {
1783  unsigned long long value;
1784  __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1785  return value;
1786 }
1787 
1788 #else /* __x86_64__ */
1789 
1790 __INTRIN_INLINE void __writecr0(unsigned int Data)
1791 {
1792  __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1793 }
1794 
1795 __INTRIN_INLINE void __writecr3(unsigned int Data)
1796 {
1797  __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1798 }
1799 
1800 __INTRIN_INLINE void __writecr4(unsigned int Data)
1801 {
1802  __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1803 }
1804 
1805 __INTRIN_INLINE unsigned long __readcr0(void)
1806 {
1807  unsigned long value;
1808  __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1809  return value;
1810 }
1811 
1812 __INTRIN_INLINE unsigned long __readcr2(void)
1813 {
1814  unsigned long value;
1815  __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1816  return value;
1817 }
1818 
1819 __INTRIN_INLINE unsigned long __readcr3(void)
1820 {
1821  unsigned long value;
1822  __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1823  return value;
1824 }
1825 
1826 __INTRIN_INLINE unsigned long __readcr4(void)
1827 {
1828  unsigned long value;
1829  __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1830  return value;
1831 }
1832 
1833 #endif /* __x86_64__ */
1834 
1835 #ifdef __x86_64__
1836 
1837 __INTRIN_INLINE unsigned long long __readdr(unsigned int reg)
1838 {
1839  unsigned long long value;
1840  switch (reg)
1841  {
1842  case 0:
1843  __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1844  break;
1845  case 1:
1846  __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1847  break;
1848  case 2:
1849  __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1850  break;
1851  case 3:
1852  __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1853  break;
1854  case 4:
1855  __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1856  break;
1857  case 5:
1858  __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1859  break;
1860  case 6:
1861  __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1862  break;
1863  case 7:
1864  __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1865  break;
1866  }
1867  return value;
1868 }
1869 
1870 __INTRIN_INLINE void __writedr(unsigned reg, unsigned long long value)
1871 {
1872  switch (reg)
1873  {
1874  case 0:
1875  __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1876  break;
1877  case 1:
1878  __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1879  break;
1880  case 2:
1881  __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1882  break;
1883  case 3:
1884  __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1885  break;
1886  case 4:
1887  __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1888  break;
1889  case 5:
1890  __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1891  break;
1892  case 6:
1893  __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1894  break;
1895  case 7:
1896  __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1897  break;
1898  }
1899 }
1900 
1901 #else /* __x86_64__ */
1902 
1903 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1904 {
1905  unsigned int value;
1906  switch (reg)
1907  {
1908  case 0:
1909  __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1910  break;
1911  case 1:
1912  __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1913  break;
1914  case 2:
1915  __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1916  break;
1917  case 3:
1918  __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1919  break;
1920  case 4:
1921  __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1922  break;
1923  case 5:
1924  __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1925  break;
1926  case 6:
1927  __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1928  break;
1929  case 7:
1930  __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1931  break;
1932  }
1933  return value;
1934 }
1935 
1936 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1937 {
1938  switch (reg)
1939  {
1940  case 0:
1941  __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1942  break;
1943  case 1:
1944  __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1945  break;
1946  case 2:
1947  __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1948  break;
1949  case 3:
1950  __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1951  break;
1952  case 4:
1953  __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1954  break;
1955  case 5:
1956  __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1957  break;
1958  case 6:
1959  __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1960  break;
1961  case 7:
1962  __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1963  break;
1964  }
1965 }
1966 
1967 #endif /* __x86_64__ */
1968 
1970 {
1971  __asm__ __volatile__ ("invlpg (%[Address])" : : [Address] "b" (Address) : "memory");
1972 }
1973 
1974 
1975 /*** System operations ***/
1976 
1977 __INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
1978 {
1979 #ifdef __x86_64__
1980  unsigned long low, high;
1981  __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1982  return ((unsigned long long)high << 32) | low;
1983 #else
1984  unsigned long long retval;
1985  __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1986  return retval;
1987 #endif
1988 }
1989 
1990 __INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
1991 {
1992 #ifdef __x86_64__
1993  __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1994 #else
1995  __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1996 #endif
1997 }
1998 
1999 __INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
2000 {
2001  unsigned long long retval;
2002  __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
2003  return retval;
2004 }
2005 
2006 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
2007 __INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
2008 {
2009  unsigned long retval;
2010  __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
2011  return retval;
2012 }
2013 
2015 {
2016  __asm__ __volatile__("wbinvd" : : : "memory");
2017 }
2018 
2020 {
2021  __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
2022 }
2023 
2025 {
2026  __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
2027 }
2028 
2030 {
2031  __asm__ __volatile__("sgdt %0" : : "m"(*(short*)Destination) : "memory");
2032 }
2033 
2034 /*** Misc operations ***/
2035 
2036 #if !HAS_BUILTIN(_mm_pause)
2038 {
2039  __asm__ __volatile__("pause" : : : "memory");
2040 }
2041 #endif
2042 
2044 {
2045  __asm__ __volatile__("nop");
2046 }
2047 
2048 #ifdef __cplusplus
2049 }
2050 #endif
2051 
2052 #endif /* KJK_INTRIN_X86_H_ */
2053 
2054 /* EOF */
__INTRIN_INLINE void __ud2(void)
Definition: intrin_x86.h:1692
CPPORT Port[4]
Definition: headless.c:35
__INTRIN_INLINE void __writecr4(unsigned int Data)
Definition: intrin_x86.h:1800
__INTRIN_INLINE unsigned short __cdecl _inpw(unsigned short Port)
Definition: intrin_x86.h:1614
__INTRIN_INLINE void __stosd(unsigned long *Dest, unsigned long Data, size_t Count)
Definition: intrin_x86.h:791
__INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
Definition: intrin_x86.h:1936
Definition: pdh_main.c:93
__INTRIN_INLINE void _mm_lfence(void)
Definition: intrin_x86.h:106
__INTRIN_INLINE void __stosb(unsigned char *Dest, unsigned char Data, size_t Count)
Definition: intrin_x86.h:770
__INTRIN_INLINE unsigned long long __cdecl _rotl64(unsigned long long value, int shift)
Definition: intrin_x86.h:1235
#define shift
Definition: input.c:1756
__INTRIN_INLINE void __addfsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:1017
__INTRIN_INLINE void *__cdecl memcpy(void *dest, const void *source, size_t num)
Definition: intrin_x86.h:74
__INTRIN_INLINE char _InterlockedCompareExchange8(volatile char *Destination, char Exchange, char Comperand)
Definition: intrin_x86.h:368
__INTRIN_INLINE unsigned long __readcr2(void)
Definition: intrin_x86.h:1812
__INTRIN_INLINE void __movsb(unsigned char *Destination, const unsigned char *Source, size_t Count)
Definition: intrin_x86.h:813
__INTRIN_INLINE unsigned long long __readmsr(unsigned long reg)
Definition: intrin_x86.h:1977
__INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long *a, long b)
Definition: intrin_x86.h:730
#define __cdecl
Definition: accygwin.h:79
__INTRIN_INLINE void __writefsbyte(unsigned long Offset, unsigned char Data)
Definition: intrin_x86.h:959
__INTRIN_INLINE unsigned short __cdecl _rotl16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1209
__INTRIN_INLINE void __incfsdword(unsigned long Offset)
Definition: intrin_x86.h:1011
__INTRIN_INLINE unsigned long __readfsdword(unsigned long Offset)
Definition: intrin_x86.h:993
__INTRIN_INLINE unsigned char _BitScanReverse(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1055
__INTRIN_INLINE void __wbinvd(void)
Definition: intrin_x86.h:2014
__INTRIN_INLINE long _InterlockedXor(volatile long *value, long mask)
Definition: intrin_x86.h:619
__INTRIN_INLINE void __incfsword(unsigned long Offset)
Definition: intrin_x86.h:1006
__INTRIN_INLINE unsigned long __cdecl _lrotr(unsigned long value, int shift)
Definition: intrin_x86.h:1297
__INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1546
__INTRIN_INLINE void __halt(void)
Definition: intrin_x86.h:1715
__INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long *Destination, long long Exchange, long long Comperand)
Definition: intrin_x86.h:692
__INTRIN_INLINE void __stosw(unsigned short *Dest, unsigned short Data, size_t Count)
Definition: intrin_x86.h:781
__INTRIN_INLINE void __lidt(void *Source)
Definition: intrin_x86.h:2019
__INTRIN_INLINE unsigned int __popcnt(unsigned int value)
Definition: intrin_x86.h:1449
GLint GLint GLint GLint GLint x
Definition: gl.h:1548
__INTRIN_INLINE unsigned long long __ull_rshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1380
__INTRIN_INLINE unsigned long __segmentlimit(unsigned long a)
Definition: intrin_x86.h:2007
__declspec(noreturn) __INTRIN_INLINE void __fastfail(unsigned int Code)
Definition: intrin_x86.h:1721
__INTRIN_INLINE unsigned long long __cdecl _byteswap_uint64(unsigned long long value)
Definition: intrin_x86.h:1417
__INTRIN_INLINE unsigned long __readcr3(void)
Definition: intrin_x86.h:1819
__INTRIN_INLINE unsigned char __readfsbyte(unsigned long Offset)
Definition: intrin_x86.h:975
__INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1568
__INTRIN_INLINE void __movsd(unsigned long *Destination, const unsigned long *Source, size_t Count)
Definition: intrin_x86.h:833
__INTRIN_INLINE long long __emul(int a, int b)
Definition: intrin_x86.h:1481
IN OUT PLONG Addend
Definition: CrNtStubs.h:22
__INTRIN_INLINE short _InterlockedIncrement16(volatile short *lpAddend)
Definition: intrin_x86.h:659
__INTRIN_INLINE void __writefsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:964
__INTRIN_INLINE unsigned int __cdecl _rotl(unsigned int value, int shift)
Definition: intrin_x86.h:1218
__INTRIN_INLINE unsigned long long __cdecl _rotr64(unsigned long long value, int shift)
Definition: intrin_x86.h:1279
__INTRIN_INLINE unsigned int __lzcnt(unsigned int value)
Definition: intrin_x86.h:1435
__INTRIN_INLINE unsigned char _BitScanForward(unsigned long *Index, unsigned long Mask)
Definition: intrin_x86.h:1047
__INTRIN_INLINE unsigned char _bittestandset(long *a, long b)
Definition: intrin_x86.h:1142
__INTRIN_INLINE long _InterlockedOr(volatile long *value, long mask)
Definition: intrin_x86.h:562
__INTRIN_INLINE int __cdecl _inp(unsigned short Port)
Definition: intrin_x86.h:1609
__INTRIN_INLINE unsigned long __cdecl _byteswap_ulong(unsigned long value)
Definition: intrin_x86.h:1402
__INTRIN_INLINE void __cdecl __debugbreak(void)
Definition: intrin_x86.h:1685
__INTRIN_INLINE void __writecr3(unsigned int Data)
Definition: intrin_x86.h:1795
__INTRIN_INLINE void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count)
Definition: intrin_x86.h:1594
__INTRIN_INLINE unsigned char _bittestandreset(long *a, long b)
Definition: intrin_x86.h:1128
GLenum GLint GLuint mask
Definition: glext.h:6028
__INTRIN_INLINE uintptr_t __readeflags(void)
Definition: intrin_x86.h:1675
__INTRIN_INLINE long _InterlockedIncrement(volatile long *lpAddend)
Definition: intrin_x86.h:645
__asm__("\t.globl GetPhys\n" "GetPhys:\t\n" "mflr 0\n\t" "stwu 0,-16(1)\n\t" "mfmsr 5\n\t" "andi. 6,5,0xffef\n\t" "mtmsr 6\n\t" "isync\n\t" "sync\n\t" "lwz 3,0(3)\n\t" "mtmsr 5\n\t" "isync\n\t" "sync\n\t" "lwz 0,0(1)\n\t" "addi 1,1,16\n\t" "mtlr 0\n\t" "blr")
__INTRIN_INLINE void __sidt(void *Destination)
Definition: intrin_x86.h:2024
__INTRIN_INLINE void _ReadWriteBarrier(void)
Definition: intrin_x86.h:88
#define _ReadBarrier
Definition: intrin_x86.h:95
__INTRIN_INLINE void __addfsword(unsigned long Offset, unsigned short Data)
Definition: intrin_x86.h:1025
unsigned int uintptr_t
Definition: crtdefs.h:304
__INTRIN_INLINE void __writefsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:969
__INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long *const Addend, const long Value)
Definition: intrin_arm.h:359
__INTRIN_INLINE unsigned short __readfsword(unsigned long Offset)
Definition: intrin_x86.h:984
__INTRIN_INLINE void __writecr0(unsigned int Data)
Definition: intrin_x86.h:1790
static WCHAR Address[46]
Definition: ping.c:68
__INTRIN_INLINE unsigned char __inbyte(unsigned short Port)
Definition: intrin_x86.h:1525
__INTRIN_INLINE char _InterlockedExchangeAdd8(char volatile *Addend, char Value)
Definition: intrin_x86.h:440
__INTRIN_INLINE unsigned char __cdecl _rotr8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1253
__INTRIN_INLINE short _InterlockedDecrement16(volatile short *lpAddend)
Definition: intrin_x86.h:652
Definition: bufpool.h:45
__INTRIN_INLINE long _InterlockedCompareExchange(volatile long *Destination, long Exchange, long Comperand)
Definition: intrin_x86.h:386
_Must_inspect_result_ _In_ WDFKEY _In_ PCUNICODE_STRING _Out_opt_ PUSHORT _Inout_opt_ PUNICODE_STRING Value
Definition: wdfregistry.h:406
#define _WriteBarrier
Definition: intrin_x86.h:96
__INTRIN_INLINE void __nop(void)
Definition: intrin_x86.h:2043
__INTRIN_INLINE long _InterlockedAnd(volatile long *value, long mask)
Definition: intrin_x86.h:505
int Count
Definition: noreturn.cpp:7
__INTRIN_INLINE short _InterlockedAnd16(volatile short *value, short mask)
Definition: intrin_x86.h:486
__INTRIN_INLINE void _mm_pause(void)
Definition: intrin_x86.h:2037
GLboolean GLboolean GLboolean b
Definition: glext.h:6204
__INTRIN_INLINE long long __cdecl _abs64(long long value)
Definition: intrin_x86.h:1498
__INTRIN_INLINE void _mm_sfence(void)
Definition: intrin_x86.h:115
__INTRIN_INLINE long long __ll_rshift(long long Mask, int Bit)
Definition: intrin_x86.h:1366
__INTRIN_INLINE char _InterlockedXor8(volatile char *value, char mask)
Definition: intrin_x86.h:581
__INTRIN_INLINE void * _InterlockedCompareExchangePointer(void *volatile *Destination, void *Exchange, void *Comperand)
Definition: intrin_x86.h:395
__INTRIN_INLINE long _InterlockedDecrement(volatile long *lpAddend)
Definition: intrin_x86.h:638
_In_ WDFCOLLECTION _In_ ULONG Index
__INTRIN_INLINE void * _InterlockedExchangePointer(void *volatile *Target, void *Value)
Definition: intrin_x86.h:431
__INTRIN_INLINE unsigned int __cdecl _rotr(unsigned int value, int shift)
Definition: intrin_x86.h:1244
__INTRIN_INLINE void __cdecl _enable(void)
Definition: intrin_x86.h:1710
__INTRIN_INLINE void __writemsr(unsigned long Register, unsigned long long Value)
Definition: intrin_x86.h:1990
GLuint GLuint num
Definition: glext.h:9618
__INTRIN_INLINE void __cpuid(int CPUInfo[4], int InfoType)
Definition: intrin_x86.h:1645
_In_ PUNICODE_STRING _Inout_ PUNICODE_STRING Destination
Definition: rtlfuncs.h:2950
__INTRIN_INLINE unsigned long __cdecl _inpd(unsigned short Port)
Definition: intrin_x86.h:1619
static int reg
Definition: i386-dis.c:1283
__INTRIN_INLINE char _InterlockedExchange8(volatile char *Target, char Value)
Definition: intrin_x86.h:404
__INTRIN_INLINE void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count)
Definition: intrin_x86.h:1604
__INTRIN_INLINE long _InterlockedExchangeAdd(volatile long *Addend, long Value)
Definition: intrin_x86.h:458
__INTRIN_INLINE void __int2c(void)
Definition: intrin_x86.h:1699
__INTRIN_INLINE long _InterlockedExchange(volatile long *Target, long Value)
Definition: intrin_x86.h:422
__INTRIN_INLINE short _InterlockedCompareExchange16(volatile short *Destination, short Exchange, short Comperand)
Definition: intrin_x86.h:377
#define __INTRIN_INLINE
Definition: intrin.h:39
__INTRIN_INLINE void __writeeflags(uintptr_t Value)
Definition: intrin_x86.h:1670
_In_ UCHAR _In_ UCHAR _In_ ULONG Code
Definition: wdfdevice.h:1697
__INTRIN_INLINE unsigned long long __rdtsc(void)
Definition: intrin_x86.h:1656
__INTRIN_INLINE void __cdecl _disable(void)
Definition: intrin_x86.h:1705
GLsizei const GLfloat * value
Definition: glext.h:6069
__INTRIN_INLINE unsigned int __readdr(unsigned int reg)
Definition: intrin_x86.h:1903
__INTRIN_INLINE char _InterlockedOr8(volatile char *value, char mask)
Definition: intrin_x86.h:524
__INTRIN_INLINE unsigned long __readcr0(void)
Definition: intrin_x86.h:1805
__INTRIN_INLINE unsigned long __readcr4(void)
Definition: intrin_x86.h:1826
processorSet Mask
__INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1557
__INTRIN_INLINE void _sgdt(void *Destination)
Definition: intrin_x86.h:2029
__INTRIN_INLINE unsigned long __indword(unsigned short Port)
Definition: intrin_x86.h:1539
__INTRIN_INLINE unsigned long long __emulu(unsigned int a, unsigned int b)
Definition: intrin_x86.h:1490
__INTRIN_INLINE unsigned long __cdecl _lrotl(unsigned long value, int shift)
Definition: intrin_x86.h:1288
#define local
Definition: zutil.h:30
_In_ ULONG _In_ ULONG Offset
Definition: ntddpcm.h:101
__INTRIN_INLINE unsigned long long __readpmc(unsigned long counter)
Definition: intrin_x86.h:1999
__INTRIN_INLINE unsigned short __lzcnt16(unsigned short value)
Definition: intrin_x86.h:1442
__INTRIN_INLINE void __cpuidex(int CPUInfo[4], int InfoType, int ECXValue)
Definition: intrin_x86.h:1650
__INTRIN_INLINE char _InterlockedAnd8(volatile char *value, char mask)
Definition: intrin_x86.h:467
#define byte(x, n)
Definition: tomcrypt.h:118
__INTRIN_INLINE void __addfsdword(unsigned long Offset, unsigned long Data)
Definition: intrin_x86.h:1033
CPUINFO CPUInfo[]
Definition: parse.c:231
__INTRIN_INLINE void __incfsbyte(unsigned long Offset)
Definition: intrin_x86.h:1001
__INTRIN_INLINE unsigned short __cdecl _outpw(unsigned short Port, unsigned short dataword)
Definition: intrin_x86.h:1630
__INTRIN_INLINE unsigned long long __ll_lshift(unsigned long long Mask, int Bit)
Definition: intrin_x86.h:1352
void *__cdecl memmove(void *dest, const void *source, size_t num)
Definition: memmove.c:8
GLint GLint GLint GLint GLint GLint y
Definition: gl.h:1548
__INTRIN_INLINE short _InterlockedOr16(volatile short *value, short mask)
Definition: intrin_x86.h:543
const WCHAR * word
Definition: lex.c:36
_In_ WDFIOTARGET Target
Definition: wdfrequest.h:306
__INTRIN_INLINE int __cdecl _outp(unsigned short Port, int databyte)
Definition: intrin_x86.h:1624
__INTRIN_INLINE short _InterlockedExchange16(volatile short *Target, short Value)
Definition: intrin_x86.h:413
__INTRIN_INLINE short _InterlockedExchangeAdd16(volatile short *Addend, short Value)
Definition: intrin_x86.h:449
__INTRIN_INLINE unsigned char __cdecl _rotl8(unsigned char value, unsigned char shift)
Definition: intrin_x86.h:1200
__INTRIN_INLINE void __invlpg(void *Address)
Definition: intrin_x86.h:1969
__INTRIN_INLINE unsigned char _bittest(const long *a, long b)
Definition: intrin_x86.h:1064
__INTRIN_INLINE unsigned short __cdecl _rotr16(unsigned short value, unsigned char shift)
Definition: intrin_x86.h:1262
__INTRIN_INLINE short _InterlockedXor16(volatile short *value, short mask)
Definition: intrin_x86.h:600
__INTRIN_INLINE unsigned short __cdecl _byteswap_ushort(unsigned short value)
Definition: intrin_x86.h:1395
__INTRIN_INLINE void _mm_mfence(void)
Definition: intrin_x86.h:99
__INTRIN_INLINE unsigned long __cdecl _outpd(unsigned short Port, unsigned long dataword)
Definition: intrin_x86.h:1636
_In_ UINT _In_ UINT _In_ PNDIS_PACKET Source
Definition: ndis.h:3167
static char * dest
Definition: rtl.c:135
__INTRIN_INLINE void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count)
Definition: intrin_x86.h:1599
GLboolean GLboolean GLboolean GLboolean a
Definition: glext.h:6204
__INTRIN_INLINE void __outword(unsigned short Port, unsigned short Data)
Definition: intrin_x86.h:1584
__INTRIN_INLINE unsigned short __inword(unsigned short Port)
Definition: intrin_x86.h:1532
__INTRIN_INLINE void __outbyte(unsigned short Port, unsigned char Data)
Definition: intrin_x86.h:1579
__INTRIN_INLINE void __movsw(unsigned short *Destination, const unsigned short *Source, size_t Count)
Definition: intrin_x86.h:823
__INTRIN_INLINE void __outdword(unsigned short Port, unsigned long Data)
Definition: intrin_x86.h:1589
__INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long *a, long b)
Definition: intrin_x86.h:749
__INTRIN_INLINE unsigned char _bittestandcomplement(long *a, long b)
Definition: intrin_x86.h:1114
__INTRIN_INLINE unsigned short __popcnt16(unsigned short value)
Definition: intrin_x86.h:1456