palacios/include/palacios/vmm_instr_emulator.h

   1 /*
   2  * This file is part of the Palacios Virtual Machine Monitor developed
   3  * by the V3VEE Project with funding from the United States National
   4  * Science Foundation and the Department of Energy.
   5  *
   6  * The V3VEE Project is a joint project between Northwestern University
   7  * and the University of New Mexico.  You can find out more at
   8  * http://www.v3vee.org
   9  *
  10  * Copyright (c) 2008, Jack Lange <jarusl@cs.northwestern.edu>
  11  * Copyright (c) 2008, The V3VEE Project <http://www.v3vee.org>
  12  * All rights reserved.
  13  *
  14  * Author: Jack Lange <jarusl@cs.northwestern.edu>
  15  *
  16  * This is free software.  You are permitted to use,
  17  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  18  */
  19
  20 #include <palacios/vmm_types.h>
  21
  22 #define FLAGS_MASK 0x00000cff
  23
  24
  25
  26 #define MAKE_1OP_8FLAGS_INST(iname) static inline void iname##8(addr_t * dst,  addr_t * flags) { \
  27         addr_t guest_flags = *flags & FLAGS_MASK;                       \
  28         uint8_t tmp_dst = *(uint8_t *)dst;                              \
  29                                                                         \
  30         asm volatile (                                                  \
  31                       "pushf; "                                         \
  32                       "push %2; "                                       \
  33                       "popf; "                                          \
  34                       #iname"b %0; "                                    \
  35                       "pushf; "                                         \
  36                       "pop %1; "                                        \
  37                       "popf; "                                          \
  38                       : "=&q"(tmp_dst),"=q"(guest_flags)                \
  39                       : "1"(guest_flags), "0"(tmp_dst)                  \
  40                       : "memory"                                        \
  41                       );                                                \
  42         *flags &= ~FLAGS_MASK;                                          \
  43         *flags |= (guest_flags & FLAGS_MASK);                           \
  44         *(uint8_t *)dst = tmp_dst;                                      \
  45                                                                         \
  46     }
  47
  48 #define MAKE_1OP_16FLAGS_INST(iname) static inline void iname##16(addr_t * dst,  addr_t * flags) { \
  49         addr_t guest_flags = *flags & FLAGS_MASK;                       \
  50         uint16_t tmp_dst = *(uint16_t *)dst;                            \
  51                                                                         \
  52         asm volatile (                                                  \
  53                       "pushf; "                                         \
  54                       "push %2; "                                       \
  55                       "popf; "                                          \
  56                       #iname"w %0; "                                    \
  57                       "pushf; "                                         \
  58                       "pop %1; "                                        \
  59                       "popf; "                                          \
  60                       : "=&q"(tmp_dst),"=q"(guest_flags)                \
  61                       : "1"(guest_flags), "0"(tmp_dst)                  \
  62                       : "memory"                                        \
  63                       );                                                \
  64         *flags &= ~FLAGS_MASK;                                          \
  65         *flags |= (guest_flags & FLAGS_MASK);                           \
  66         *(uint16_t *)dst = tmp_dst;                                     \
  67     }
  68
  69 #define MAKE_1OP_32FLAGS_INST(iname) static inline void iname##32(addr_t * dst,  addr_t * flags) { \
  70         addr_t guest_flags = *flags & FLAGS_MASK;                       \
  71         uint32_t tmp_dst = *(uint32_t *)dst;                            \
  72                                                                         \
  73         asm volatile (                                                  \
  74                       "pushf; "                                         \
  75                       "push %2; "                                       \
  76                       "popf; "                                          \
  77                       #iname"l %0; "                                    \
  78                       "pushf; "                                         \
  79                       "pop %1; "                                        \
  80                       "popf; "                                          \
  81                       : "=&q"(tmp_dst),"=q"(guest_flags)                \
  82                       : "1"(guest_flags), "0"(tmp_dst)                  \
  83                       : "memory"                                        \
  84                       );                                                \
  85                                                                         \
  86         *flags &= ~FLAGS_MASK;                                          \
  87         *flags |= (guest_flags & FLAGS_MASK);                           \
  88         *(uint32_t *)dst = tmp_dst;                                     \
  89     }
  90
  91 #define MAKE_1OP_64FLAGS_INST(iname) static inline void iname##64(addr_t * dst,  addr_t * flags) { \
  92         addr_t guest_flags = *flags & FLAGS_MASK;                       \
  93         uint64_t tmp_dst = *(uint64_t *)dst;                            \
  94                                                                         \
  95         asm volatile (                                                  \
  96                       "pushfq; "                                        \
  97                       "push %2; "                                       \
  98                       "popfq; "                                         \
  99                       #iname"q %0; "                                    \
 100                       "pushfq; "                                        \
 101                       "pop %1; "                                        \
 102                       "popfq; "                                         \
 103                       : "=q"(tmp_dst),"=q"(guest_flags)                 \
 104                       : "q"(guest_flags), "0"(tmp_dst)                  \
 105                       : "memory"                                        \
 106                       );                                                \
 107                                                                         \
 108         *flags &= ~FLAGS_MASK;                                          \
 109         *flags |= (guest_flags & FLAGS_MASK);                           \
 110         *(uint64_t *)dst = tmp_dst;                                     \
 111     }
 112
 113
 114
 115 #define MAKE_1OP_8_INST(iname) static inline void iname##8(addr_t * dst) { \
 116         uint8_t tmp_dst = *(uint8_t *)dst;                              \
 117         asm volatile (                                                  \
 118                       #iname"b %0; "                                    \
 119                       : "=q"(tmp_dst)                                   \
 120                       : "0"(tmp_dst)                                    \
 121                       : "memory"                                        \
 122                       );                                                \
 123         *(uint8_t *)dst = tmp_dst;                                      \
 124     }
 125
 126 #define MAKE_1OP_16_INST(iname) static inline void iname##16(addr_t * dst) { \
 127         uint16_t tmp_dst = *(uint16_t *)dst;                            \
 128         asm volatile (                                                  \
 129                       #iname"w %0; "                                    \
 130                       : "=q"(tmp_dst)                                   \
 131                       :  "0"(tmp_dst)                                   \
 132                       : "memory"                                        \
 133                       );                                                \
 134         *(uint16_t *)dst = tmp_dst;                                     \
 135     }
 136
 137 #define MAKE_1OP_32_INST(iname) static inline void iname##32(addr_t * dst) { \
 138         uint32_t tmp_dst = *(uint32_t *)dst;                            \
 139         asm volatile (                                                  \
 140                       #iname"l %0; "                                    \
 141                       : "=q"(tmp_dst)                                   \
 142                       : "0"(tmp_dst)                                    \
 143                       : "memory"                                        \
 144                       );                                                \
 145         *(uint32_t *)dst = tmp_dst;                                     \
 146     }
 147
 148 #define MAKE_1OP_64_INST(iname) static inline void iname##64(addr_t * dst) { \
 149         uint64_t tmp_dst = *(uint64_t *)dst;                            \
 150         asm volatile (                                                  \
 151                       #iname"q %0; "                                    \
 152                       : "=q"(tmp_dst)                                   \
 153                       : "0"(tmp_dst)                                    \
 154                       : "memory"                                        \
 155                       );                                                \
 156         *(uint64_t *)dst = tmp_dst;                                     \
 157     }
 158
 159
 160 #define MAKE_2OP_64FLAGS_INST(iname) static inline void iname##64(addr_t * dst, addr_t * src, addr_t * flags) { \
 161         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 162         uint64_t tmp_dst = *(uint64_t *)dst;                            \
 163                                                                         \
 164         asm volatile (                                                  \
 165                       "pushfq\r\n"                                      \
 166                       "push %3\r\n"                                     \
 167                       "popfq\r\n"                                       \
 168                       #iname"q %2, %0\r\n"                              \
 169                       "pushfq\r\n"                                      \
 170                       "pop %1\r\n"                                      \
 171                       "popfq\r\n"                                       \
 172                       : "=&q"(tmp_dst),"=q"(guest_flags)                        \
 173                       : "q"(*(uint64_t *)src),"1"(guest_flags), "0"(tmp_dst) \
 174                       : "memory"                                        \
 175                       );                                                \
 176         *flags &= ~FLAGS_MASK;                                          \
 177         *flags |= (guest_flags & FLAGS_MASK);                           \
 178         *(uint64_t *)dst = tmp_dst;                                     \
 179     }
 180
 181
 182
 183
 184 #define MAKE_2OP_32FLAGS_INST(iname) static inline void iname##32(addr_t * dst, addr_t * src, addr_t * flags) { \
 185         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 186         uint32_t tmp_dst = *(uint32_t *)dst;                            \
 187                                                                         \
 188         asm volatile (                                                  \
 189                       "pushf; "                                         \
 190                       "push %3; "                                       \
 191                       "popf; "                                          \
 192                       #iname"l %2, %0; "                                \
 193                       "pushf; "                                         \
 194                       "pop %1; "                                        \
 195                       "popf; "                                          \
 196                       : "=&q"(tmp_dst),"=q"(guest_flags)                \
 197                       : "q"(*(uint32_t *)src),"1"(guest_flags), "0"(tmp_dst) \
 198                       : "memory"                                        \
 199                       );                                                \
 200         *flags &= ~FLAGS_MASK;                                          \
 201         *flags |= (guest_flags & FLAGS_MASK);                           \
 202         *(uint32_t *)dst = tmp_dst;                                     \
 203     }
 204
 205
 206 #define MAKE_2OP_16FLAGS_INST(iname) static inline void iname##16(addr_t * dst, addr_t * src, addr_t * flags) { \
 207         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 208         uint16_t tmp_dst = *(uint16_t *)dst;                            \
 209                                                                         \
 210         asm volatile (                                                  \
 211                       "pushf; "                                         \
 212                       "push %3; "                                       \
 213                       "popf; "                                          \
 214                       #iname"w %2, %0; "                                \
 215                       "pushf; "                                         \
 216                       "pop %1; "                                        \
 217                       "popf; "                                          \
 218                       : "=&q"(tmp_dst),"=q"(guest_flags)                \
 219                       : "q"(*(uint16_t *)src),"1"(guest_flags), "0"(tmp_dst) \
 220                       : "memory"                                        \
 221                       );                                                \
 222         *flags &= ~FLAGS_MASK;                                          \
 223         *flags |= (guest_flags & FLAGS_MASK);                           \
 224         *(uint16_t *)dst = tmp_dst;                                     \
 225     }
 226
 227 #define MAKE_2OP_8FLAGS_INST(iname) static inline void iname##8(addr_t * dst, addr_t * src, addr_t * flags) { \
 228         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 229         uint8_t tmp_dst = *(uint8_t *)dst;                              \
 230                                                                         \
 231         asm volatile (                                                  \
 232                       "pushf; "                                         \
 233                       "push %3; "                                       \
 234                       "popf; "                                          \
 235                       #iname"b %2, %0; "                                \
 236                       "pushf; "                                         \
 237                       "pop %1; "                                        \
 238                       "popf; "                                          \
 239                       : "=q"(tmp_dst),"=q"(guest_flags)                 \
 240                       : "q"(*(uint8_t *)src),"1"(guest_flags), "0"(tmp_dst) \
 241                       : "memory"                                        \
 242                       );                                                \
 243         *flags &= ~FLAGS_MASK;                                          \
 244         *flags |= (guest_flags & FLAGS_MASK);                           \
 245         *(uint8_t *)dst = tmp_dst;                                      \
 246     }
 247
 248
 249
 250
 251
 252 #define MAKE_2OP_64STR_INST(iname) static inline void iname##64(addr_t * dst, \
 253                                                                 addr_t * src, \
 254                                                                 addr_t * ecx, addr_t * flags) { \
 255         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 256         uint64_t tmp_dst = 0, tmp_ecx = 0, tmp_src = 0;                 \
 257                                                                         \
 258         asm volatile (                                                  \
 259                       "pushfq; "                                        \
 260                       "pushq %7; "                                      \
 261                       "popfq; "                                         \
 262                       "rep; "                                           \
 263                       #iname"q; "                                       \
 264                       "pushfq; "                                        \
 265                       "popq %0; "                                       \
 266                       "popfq; "                                         \
 267                       : "=q"(guest_flags), "=&D"(tmp_dst), "=&S"(tmp_src), "=&c"(tmp_ecx) \
 268                       : "1"(*dst),"2"(*src),"3"(*ecx),"0"(guest_flags)  \
 269                       : "memory"                                        \
 270                       );                                                \
 271         *flags &= ~FLAGS_MASK;                                          \
 272         *flags |= (guest_flags & FLAGS_MASK);                           \
 273     }
 274
 275
 276 #define MAKE_2OP_32STR_INST(iname) static inline void iname##32(addr_t * dst, \
 277                                                                 addr_t * src, \
 278                                                                 addr_t * ecx, addr_t * flags) { \
 279         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 280         uint32_t tmp_dst = 0, tmp_ecx = 0, tmp_src = 0;                 \
 281                                                                         \
 282         asm volatile (                                                  \
 283                       "pushf; "                                         \
 284                       "push %7; "                                       \
 285                       "popf; "                                          \
 286                       "rep; "                                           \
 287                       #iname"l; "                                       \
 288                       "pushf; "                                         \
 289                       "pop %0; "                                        \
 290                       "popf; "                                          \
 291                       : "=q"(guest_flags), "=&D"(tmp_dst), "=&S"(tmp_src), "=&c"(tmp_ecx) \
 292                       : "1"(*dst),"2"(*src),"3"(*ecx),"0"(guest_flags)  \
 293                       : "memory"                                        \
 294                       );                                                \
 295         *flags &= ~FLAGS_MASK;                                          \
 296         *flags |= (guest_flags & FLAGS_MASK);                           \
 297     }
 298
 299 #define MAKE_2OP_16STR_INST(iname) static inline void iname##16(addr_t * dst, \
 300                                                                 addr_t * src, \
 301                                                                 addr_t * ecx, addr_t * flags) { \
 302         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 303         uint16_t tmp_dst = 0, tmp_ecx = 0, tmp_src = 0;                 \
 304                                                                         \
 305         asm volatile (                                                  \
 306                       "pushf; "                                         \
 307                       "push %7; "                                       \
 308                       "popf; "                                          \
 309                       "rep; "                                           \
 310                       #iname"w; "                                       \
 311                       "pushf; "                                         \
 312                       "pop %0; "                                        \
 313                       "popf; "                                          \
 314                       : "=q"(guest_flags), "=&D"(tmp_dst), "=&S"(tmp_src), "=&c"(tmp_ecx) \
 315                       : "1"(*dst),"2"(*src),"3"(*ecx),"0"(guest_flags)  \
 316                       : "memory"                                        \
 317                       );                                                \
 318         *flags &= ~FLAGS_MASK;                                          \
 319         *flags |= (guest_flags & FLAGS_MASK);                           \
 320     }
 321
 322
 323
 324 #define MAKE_2OP_8STR_INST(iname) static inline void iname##8(addr_t * dst, \
 325                                                               addr_t * src, \
 326                                                               addr_t * ecx, addr_t * flags) { \
 327         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 328         uint8_t tmp_dst = 0, tmp_ecx = 0, tmp_src = 0;                  \
 329                                                                         \
 330         asm volatile (                                                  \
 331                       "pushf; "                                         \
 332                       "push %7; "                                       \
 333                       "popf; "                                          \
 334                       "rep; "                                           \
 335                       #iname"b; "                                       \
 336                       "pushf; "                                         \
 337                       "pop %0; "                                        \
 338                       "popf; "                                          \
 339                       : "=q"(guest_flags), "=&D"(tmp_dst), "=&S"(tmp_src), "=&c"(tmp_ecx) \
 340                       : "1"(*dst),"2"(*src),"3"(*ecx),"0"(guest_flags)  \
 341                       : "memory"                                        \
 342                       );                                                \
 343         *flags &= ~FLAGS_MASK;                                          \
 344         *flags |= (guest_flags & FLAGS_MASK);                           \
 345     }
 346
 347
 348
 349
 350 #define MAKE_1OP_64STR_INST(iname) static inline void iname##64(addr_t * dst, \
 351                                                                 addr_t * src, \
 352                                                                 addr_t * ecx, addr_t * flags) { \
 353         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 354         uint64_t tmp_dst = 0, tmp_ecx = 0;                              \
 355                                                                         \
 356         asm volatile (                                                  \
 357                       "pushfq; "                                        \
 358                       "pushq %6; "                                      \
 359                       "popfq; "                                         \
 360                       "rep; "                                           \
 361                       #iname"q; "                                       \
 362                       "pushfq; "                                        \
 363                       "popq %0; "                                       \
 364                       "popfq; "                                         \
 365                       : "=&q"(guest_flags), "=&D"(tmp_dst), "=&c"(tmp_ecx) \
 366                       : "1"(*dst),"a"(*src),"2"(*ecx),"0"(guest_flags)  \
 367                       : "memory"                                        \
 368                       );                                                \
 369                                                                         \
 370         *flags &= ~FLAGS_MASK;                                          \
 371         *flags |= (guest_flags & FLAGS_MASK);                           \
 372     }
 373
 374
 375 #define MAKE_1OP_32STR_INST(iname) static inline void iname##32(addr_t * dst, \
 376                                                                 addr_t * src, \
 377                                                                 addr_t * ecx, addr_t * flags) { \
 378         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 379         uint32_t tmp_dst = 0, tmp_ecx = 0;                              \
 380                                                                         \
 381         asm volatile (                                                  \
 382                       "pushf; "                                         \
 383                       "push %6; "                                       \
 384                       "popf; "                                          \
 385                       "rep; "                                           \
 386                       #iname"l; "                                       \
 387                       "pushf; "                                         \
 388                       "pop %0; "                                        \
 389                       "popf; "                                          \
 390                       : "=&q"(guest_flags), "=&D"(tmp_dst), "=&c"(tmp_ecx) \
 391                       : "1"(*dst),"a"(*src),"2"(*ecx),"0"(guest_flags)  \
 392                       : "memory"                                        \
 393                       );                                                \
 394                                                                         \
 395         *flags &= ~FLAGS_MASK;                                          \
 396         *flags |= (guest_flags & FLAGS_MASK);                           \
 397     }
 398
 399 #define MAKE_1OP_16STR_INST(iname) static inline void iname##16(addr_t * dst, \
 400                                                                 addr_t * src, \
 401                                                                 addr_t * ecx, addr_t * flags) { \
 402         addr_t guest_flags = *flags & FLAGS_MASK;                       \
 403         uint16_t tmp_dst = 0, tmp_ecx = 0;                              \
 404                                                                         \
 405         asm volatile (                                                  \
 406                       "pushf; "                                         \
 407                       "push %6; "                                       \
 408                       "popf; "                                          \
 409                       "rep; "                                           \
 410                       #iname"w; "                                       \
 411                       "pushf; "                                         \
 412                       "pop %0; "                                        \
 413                       "popf; "                                          \
 414                       : "=q"(guest_flags), "=&D"(tmp_dst), "=&c"(tmp_ecx) \
 415                       : "1"(*dst),"a"(*src),"2"(*ecx),"0"(guest_flags)  \
 416                       : "memory"                                        \
 417                       );                                                \
 418         *flags &= ~FLAGS_MASK;                                          \
 419         *flags |= (guest_flags & FLAGS_MASK);                           \
 420     }
 421
 422
 423
 424 #define MAKE_1OP_8STR_INST(iname) static inline void iname##8(addr_t * dst, \
 425                                                               addr_t * src, \
 426                                                               addr_t * ecx, addr_t * flags) { \
 427         /* Some of the flags values are not copied out in a pushf, we save them here */ \
 428         addr_t guest_flags = *flags & 0x00000cff;                       \
 429         uint8_t tmp_dst = 0, tmp_ecx = 0;                               \
 430                                                                         \
 431         asm volatile (                                                  \
 432                       "pushf; "                                         \
 433                       "push %6; "                                       \
 434                       "popf; "                                          \
 435                       "rep; "                                           \
 436                       #iname"b; "                                       \
 437                       "pushf; "                                         \
 438                       "pop %0; "                                        \
 439                       "popf; "                                          \
 440                       : "=&q"(guest_flags), "=&D"(tmp_dst), "=&c"(tmp_ecx) \
 441                       : "1"(*dst),"a"(*src),"2"(*ecx),"0"(guest_flags)  \
 442                       : "memory"                                        \
 443                       );                                                \
 444         *flags &= ~FLAGS_MASK;                                          \
 445         *flags |= (guest_flags & FLAGS_MASK);                           \
 446     }
 447
 448
 449
 450
 451 #define MAKE_2OP_64_INST(iname) static inline void iname##64(addr_t * dst, addr_t * src) { \
 452         uint64_t tmp_dst = *(uint64_t *)dst;                            \
 453         asm volatile (                                                  \
 454                       #iname"q %1, %0; "                                \
 455                       : "=&q"(tmp_dst)                                  \
 456                       : "q"(*(uint64_t *)src), "0"(tmp_dst)             \
 457                       : "memory"                                        \
 458                       );                                                \
 459         *(uint64_t *)dst = tmp_dst;                                     \
 460     }
 461
 462 #define MAKE_2OP_32_INST(iname) static inline void iname##32(addr_t * dst, addr_t * src) { \
 463         uint32_t tmp_dst = *(uint32_t *)dst;                            \
 464         asm volatile (                                                  \
 465                       #iname"l %1, %0; "                                \
 466                       : "=&q"(tmp_dst)                                  \
 467                       : "q"(*(uint32_t *)src), "0"(tmp_dst)             \
 468                       : "memory"                                        \
 469                       );                                                \
 470         *(uint32_t *)dst = tmp_dst;                                     \
 471    }
 472
 473 #define MAKE_2OP_16_INST(iname) static inline void iname##16(addr_t * dst, addr_t * src) { \
 474         uint16_t tmp_dst = *(uint16_t *)dst;                            \
 475         asm volatile (                                                  \
 476                       #iname"w %1, %0; "                                \
 477                       : "=&q"(tmp_dst)                                  \
 478                       : "q"(*(uint16_t *)src), "0"(tmp_dst)             \
 479                       : "memory"                                        \
 480                       );                                                \
 481         *(uint16_t *)dst = tmp_dst;                                     \
 482     }
 483
 484 #define MAKE_2OP_8_INST(iname) static inline void iname##8(addr_t * dst, addr_t * src) { \
 485         uint8_t tmp_dst = *(uint8_t *)dst;                              \
 486         asm volatile (                                                  \
 487                       #iname"b %1, %0; "                                \
 488                       : "=&q"(tmp_dst)                                  \
 489                       : "q"(*(uint8_t *)src), "0"(tmp_dst)              \
 490                       : "memory"                                        \
 491                       );                                                \
 492         *(uint8_t *)dst = tmp_dst;                                      \
 493     }
 494
 495
 496
 497
 498
 499 #define MAKE_2OP_8EXT_INST(iname) static inline void iname##8(addr_t * dst, addr_t * src, uint_t dst_len) { \
 500         if (dst_len == 2) {                                             \
 501             uint16_t tmp_dst = *(uint16_t *)dst;                        \
 502             asm volatile (                                              \
 503                           #iname" %1, %0; "                             \
 504                           : "=&q"(tmp_dst)                              \
 505                           : "q"(*(uint8_t *)src), "0"(tmp_dst)          \
 506                           : "memory"                                    \
 507                           );                                            \
 508             *(uint16_t *)dst = tmp_dst;                                 \
 509         } else if (dst_len == 4) {                                      \
 510             uint32_t tmp_dst = *(uint32_t *)dst;                        \
 511             asm volatile (                                              \
 512                           #iname" %1, %0; "                             \
 513                           : "=&q"(tmp_dst)                              \
 514                           : "q"(*(uint8_t *)src), "0"(tmp_dst)          \
 515                           : "memory"                                    \
 516                           );                                            \
 517             *(uint32_t *)dst = tmp_dst;                                 \
 518         } else if (dst_len == 8) {                                      \
 519             uint64_t tmp_dst = *(uint64_t *)dst;                        \
 520             asm volatile (                                              \
 521                           #iname" %1, %0; "                             \
 522                           : "=&q"(tmp_dst)                              \
 523                           : "q"(*(uint8_t *)src), "0"(tmp_dst)          \
 524                           : "memory"                                    \
 525                           );                                            \
 526             *(uint64_t *)dst = tmp_dst;                                 \
 527         }                                                               \
 528     }
 529
 530 #define MAKE_2OP_16EXT_INST(iname) static inline void iname##16(addr_t * dst, addr_t * src, uint_t dst_len) { \
 531         if (dst_len == 4) {                                             \
 532             uint32_t tmp_dst = *(uint32_t *)dst;                        \
 533             asm volatile (                                              \
 534                           #iname" %1, %0; "                             \
 535                           : "=&q"(tmp_dst)                              \
 536                           : "q"(*(uint16_t *)src), "0"(tmp_dst)         \
 537                           : "memory"                                    \
 538                           );                                            \
 539             *(uint32_t *)dst = tmp_dst;                                 \
 540         } else if (dst_len == 8) {                                      \
 541             uint64_t tmp_dst = *(uint64_t *)dst;                        \
 542             asm volatile (                                              \
 543                           #iname" %1, %0; "                             \
 544                           : "=&q"(tmp_dst)                              \
 545                           : "q"(*(uint16_t *)src), "0"(tmp_dst)         \
 546                           : "memory"                                    \
 547                           );                                            \
 548             *(uint64_t *)dst = tmp_dst;                                 \
 549         }                                                               \
 550     }
 551
 552
 553
 554
 555 #define MAKE_2OUT_64_INST(iname) static inline void iname##64(addr_t * dst, addr_t * src) { \
 556         uint64_t tmp_dst = *(uint64_t *)dst;                            \
 557         uint64_t tmp_src = *(uint64_t *)src;                            \
 558                                                                         \
 559         asm volatile (                                                  \
 560                       #iname"q %1, %0; "                                \
 561                       : "=&q"(tmp_dst), "=&q"(tmp_src)                  \
 562                       : "0"(tmp_dst), "1"(tmp_src)                      \
 563                       : "memory"                                        \
 564                       );                                                \
 565         *(uint64_t *)src = tmp_src;                                     \
 566         *(uint64_t *)dst = tmp_dst;                                     \
 567     }
 568
 569 #define MAKE_2OUT_32_INST(iname) static inline void iname##32(addr_t * dst, addr_t * src) { \
 570         uint32_t tmp_dst = *(uint32_t *)dst;                            \
 571         uint32_t tmp_src = *(uint32_t *)src;                            \
 572                                                                         \
 573         asm volatile (                                                  \
 574                       #iname"l %1, %0; "                                \
 575                       : "=&q"(tmp_dst), "=&q"(tmp_src)                  \
 576                       :  "0"(tmp_dst), "1"(tmp_src)                     \
 577                       : "memory"                                        \
 578                       );                                                \
 579         *(uint32_t *)src = tmp_src;                                     \
 580         *(uint32_t *)dst = tmp_dst;                                     \
 581     }
 582
 583 #define MAKE_2OUT_16_INST(iname) static inline void iname##16(addr_t * dst, addr_t * src) { \
 584         uint16_t tmp_dst = *(uint16_t *)dst;                            \
 585         uint16_t tmp_src = *(uint16_t *)src;                            \
 586                                                                         \
 587         asm volatile (                                                  \
 588                       #iname"w %1, %0; "                                \
 589                       : "=&q"(tmp_dst), "=&q"(tmp_src)                  \
 590                       : "0"(tmp_dst), "1"(tmp_src)                      \
 591                       : "memory"                                        \
 592                       );                                                \
 593         *(uint16_t *)src = tmp_src;                                     \
 594         *(uint16_t *)dst = tmp_dst;                                     \
 595     }
 596
 597 #define MAKE_2OUT_8_INST(iname) static inline void iname##8(addr_t * dst, addr_t * src) { \
 598         uint8_t tmp_dst = *(uint8_t *)dst;                              \
 599         uint8_t tmp_src = *(uint8_t *)src;                              \
 600                                                                         \
 601         asm volatile (                                                  \
 602                       #iname"b %1, %0; "                                \
 603                       : "=&q"(tmp_dst), "=&q"(tmp_src)                  \
 604                       : "0"(tmp_dst), "1"(tmp_src)                      \
 605                       : "memory"                                        \
 606                       );                                                \
 607         *(uint8_t *)src = tmp_src;                                      \
 608         *(uint8_t *)dst = tmp_dst;                                      \
 609     }
 610
 611
 612
 613
 614
 615
 616 /****************************/
 617 /* 8 Bit instruction forms  */
 618 /****************************/
 619
 620 MAKE_2OP_8FLAGS_INST(adc);
 621 MAKE_2OP_8FLAGS_INST(add);
 622 MAKE_2OP_8FLAGS_INST(and);
 623 MAKE_2OP_8FLAGS_INST(or);
 624 MAKE_2OP_8FLAGS_INST(xor);
 625 MAKE_2OP_8FLAGS_INST(sub);
 626
 627
 628 MAKE_1OP_8FLAGS_INST(inc);
 629 MAKE_1OP_8FLAGS_INST(dec);
 630 MAKE_1OP_8FLAGS_INST(neg);
 631 MAKE_1OP_8FLAGS_INST(setb);
 632 MAKE_1OP_8FLAGS_INST(setbe);
 633 MAKE_1OP_8FLAGS_INST(setl);
 634 MAKE_1OP_8FLAGS_INST(setle);
 635 MAKE_1OP_8FLAGS_INST(setnb);
 636 MAKE_1OP_8FLAGS_INST(setnbe);
 637 MAKE_1OP_8FLAGS_INST(setnl);
 638 MAKE_1OP_8FLAGS_INST(setnle);
 639 MAKE_1OP_8FLAGS_INST(setno);
 640 MAKE_1OP_8FLAGS_INST(setnp);
 641 MAKE_1OP_8FLAGS_INST(setns);
 642 MAKE_1OP_8FLAGS_INST(setnz);
 643 MAKE_1OP_8FLAGS_INST(seto);
 644 MAKE_1OP_8FLAGS_INST(setp);
 645 MAKE_1OP_8FLAGS_INST(sets);
 646 MAKE_1OP_8FLAGS_INST(setz);
 647
 648
 649 MAKE_1OP_8_INST(not);
 650
 651 MAKE_2OP_8_INST(mov);
 652 MAKE_2OP_8EXT_INST(movzx);
 653 MAKE_2OP_8EXT_INST(movsx);
 654
 655 MAKE_2OUT_8_INST(xchg);
 656
 657 MAKE_2OP_8STR_INST(movs);
 658 MAKE_1OP_8STR_INST(stos);
 659 MAKE_1OP_8STR_INST(scas);
 660
 661
 662 /****************************/
 663 /* 16 Bit instruction forms */
 664 /****************************/
 665 MAKE_2OP_16FLAGS_INST(adc);
 666 MAKE_2OP_16FLAGS_INST(add);
 667 MAKE_2OP_16FLAGS_INST(and);
 668 MAKE_2OP_16FLAGS_INST(or);
 669 MAKE_2OP_16FLAGS_INST(xor);
 670 MAKE_2OP_16FLAGS_INST(sub);
 671
 672
 673 MAKE_1OP_16FLAGS_INST(inc);
 674 MAKE_1OP_16FLAGS_INST(dec);
 675 MAKE_1OP_16FLAGS_INST(neg);
 676
 677 MAKE_1OP_16_INST(not);
 678
 679 MAKE_2OP_16_INST(mov);
 680 MAKE_2OP_16EXT_INST(movzx);
 681 MAKE_2OP_16EXT_INST(movsx);
 682 MAKE_2OUT_16_INST(xchg);
 683
 684 MAKE_2OP_16STR_INST(movs);
 685 MAKE_1OP_16STR_INST(stos);
 686 MAKE_1OP_16STR_INST(scas);
 687
 688
 689 /****************************/
 690 /* 32 Bit instruction forms */
 691 /****************************/
 692 MAKE_2OP_32FLAGS_INST(adc);
 693 MAKE_2OP_32FLAGS_INST(add);
 694 MAKE_2OP_32FLAGS_INST(and);
 695 MAKE_2OP_32FLAGS_INST(or);
 696 MAKE_2OP_32FLAGS_INST(xor);
 697 MAKE_2OP_32FLAGS_INST(sub);
 698
 699
 700 MAKE_1OP_32FLAGS_INST(inc);
 701 MAKE_1OP_32FLAGS_INST(dec);
 702 MAKE_1OP_32FLAGS_INST(neg);
 703
 704 MAKE_1OP_32_INST(not);
 705
 706 MAKE_2OP_32_INST(mov);
 707
 708 MAKE_2OUT_32_INST(xchg);
 709
 710
 711
 712 MAKE_2OP_32STR_INST(movs);
 713 MAKE_1OP_32STR_INST(stos);
 714 MAKE_1OP_32STR_INST(scas);
 715
 716
 717
 718 #ifdef __V3_64BIT__
 719
 720 /****************************/
 721 /* 64 Bit instruction forms */
 722 /****************************/
 723 MAKE_2OP_64FLAGS_INST(adc);
 724 MAKE_2OP_64FLAGS_INST(add);
 725 MAKE_2OP_64FLAGS_INST(and);
 726 MAKE_2OP_64FLAGS_INST(or);
 727 MAKE_2OP_64FLAGS_INST(xor);
 728 MAKE_2OP_64FLAGS_INST(sub);
 729
 730 MAKE_1OP_64FLAGS_INST(inc);
 731 MAKE_1OP_64FLAGS_INST(dec);
 732 MAKE_1OP_64FLAGS_INST(neg);
 733
 734 MAKE_1OP_64_INST(not);
 735
 736
 737 MAKE_2OP_64_INST(mov);
 738 MAKE_2OP_64STR_INST(movs);
 739 MAKE_1OP_64STR_INST(stos);
 740 MAKE_1OP_64STR_INST(scas);
 741
 742 MAKE_2OUT_64_INST(xchg);
 743
 744
 745 #endif