Support decoding a bunch of MMX/SSE instructions

This is actually required for an upcoming feature to work, as an
instruction scan bumps into a xorps instruction.

Still not supporting SSE3 or later, or AVX, although at this point it
seems that that would be relatively easy to add, as this thing has kept
growing on an incremental case-by-case basis.

1 files changed, 202 insertions, 88 deletions

diff --git a/src/x86.h b/src/x86.h
index 46a34b2..197d4c3 100644
--- a/src/x86.h
+++ b/src/x86.h
@@ -1,5 +1,5 @@
 /*
- * Copyright © 2022 Michael Smith <mikesmiffy128@gmail.com>
+ * Copyright © 2023 Michael Smith <mikesmiffy128@gmail.com>
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -20,8 +20,8 @@
 /*
  * Opcode-based X86 instruction analysis. In other words, *NOT* a disassembler.
  * Only cares about the instructions we expect to see in basic 32-bit userspace
- * functions; there's no kernel-mode instructions, no MMX/3DNow!/SSE/AVX, no
- * REX, EVEX, yadda yadda.
+ * functions; there's no kernel-mode instructions, no SSE 3+, no AVX, no REX,
+ * EVEX, yadda yadda.
  */
 
 // XXX: no BOUND (0x62): ambiguous with EVEX prefix - can't be arsed!
@@ -316,106 +316,220 @@
 	X(X86_2B_PUSHGS,   0xA8) \
 	X(X86_2B_POPGS,    0xA9) \
 	X(X86_2B_RSM,      0xAA) \
-	X(X86_2B_BSWAPEAX,  0xC8) \
-	X(X86_2B_BSWAPECX,  0xC9) \
-	X(X86_2B_BSWAPEDX,  0xCA) \
-	X(X86_2B_BSWAPEBX,  0xCB) \
-	X(X86_2B_BSWAPESP,  0xCC) \
-	X(X86_2B_BSWAPEBP,  0xCD) \
-	X(X86_2B_BSWAPESI,  0xCE) \
-	X(X86_2B_BSWAPEDI,  0xCF)
+	X(X86_2B_BSWAPEAX, 0xC8) \
+	X(X86_2B_BSWAPECX, 0xC9) \
+	X(X86_2B_BSWAPEDX, 0xCA) \
+	X(X86_2B_BSWAPEBX, 0xCB) \
+	X(X86_2B_BSWAPESP, 0xCC) \
+	X(X86_2B_BSWAPEBP, 0xCD) \
+	X(X86_2B_BSWAPESI, 0xCE) \
+	X(X86_2B_BSWAPEDI, 0xCF) \
+	/* MMX instruction */ \
+	X(X86_2B_EMMS,     0x77)
 
 /* Second bytes of opcodes with a word-sized immediate operand */
 #define X86_OPS_2BYTE_IW(X) \
-	X(X86_2B_JOII,  0x80) /* From offset (indirect) */ \
-	X(X86_2B_JNOII, 0x81) /* From offset (indirect) */ \
-	X(X86_2B_JBII,  0x82) /* AKA JC; from offset (indirect) */ \
-	X(X86_2B_JNBII, 0x83) /* AKA JNC; from offset (indirect) */ \
-	X(X86_2B_JZII,  0x84) /* AKA JE; from offset (indirect) */ \
-	X(X86_2B_JNZII, 0x85) /* AKA JNZ; from offset (indirect) */ \
-	X(X86_2B_JNAII, 0x86) /* AKA JBE; from offset (indirect) */ \
-	X(X86_2B_JAII,  0x87) /* AKA JNBE; from offset (indirect) */ \
-	X(X86_2B_JSII,  0x88) /* From offset (indirect) */ \
-	X(X86_2B_JNSII, 0x89) /* From offset (indirect) */ \
-	X(X86_2B_JPII,  0x8A) /* From offset (indirect) */ \
-	X(X86_2B_JNPII, 0x8B) /* From offset (indirect) */ \
-	X(X86_2B_JLII,  0x8C) /* AKA JNGE; from offset (indirect) */ \
-	X(X86_2B_JNLII, 0x8D) /* AKA JGE; from offset (indirect) */ \
-	X(X86_2B_JNGII, 0x8E) /* AKA JLE; from offset (indirect) */ \
-	X(X86_2B_JGII,  0x8F) /* AKA JNLE; from offset (indirect) */
+	X(X86_2B_JOII,   0x80) /* From offset (indirect) */ \
+	X(X86_2B_JNOII,  0x81) /* From offset (indirect) */ \
+	X(X86_2B_JBII,   0x82) /* AKA JC; from offset (indirect) */ \
+	X(X86_2B_JNBII,  0x83) /* AKA JNC; from offset (indirect) */ \
+	X(X86_2B_JZII,   0x84) /* AKA JE; from offset (indirect) */ \
+	X(X86_2B_JNZII,  0x85) /* AKA JNZ; from offset (indirect) */ \
+	X(X86_2B_JNAII,  0x86) /* AKA JBE; from offset (indirect) */ \
+	X(X86_2B_JAII,   0x87) /* AKA JNBE; from offset (indirect) */ \
+	X(X86_2B_JSII,   0x88) /* From offset (indirect) */ \
+	X(X86_2B_JNSII,  0x89) /* From offset (indirect) */ \
+	X(X86_2B_JPII,   0x8A) /* From offset (indirect) */ \
+	X(X86_2B_JNPII,  0x8B) /* From offset (indirect) */ \
+	X(X86_2B_JLII,   0x8C) /* AKA JNGE; from offset (indirect) */ \
+	X(X86_2B_JNLII,  0x8D) /* AKA JGE; from offset (indirect) */ \
+	X(X86_2B_JNGII,  0x8E) /* AKA JLE; from offset (indirect) */ \
+	X(X86_2B_JGII,   0x8F) /* AKA JNLE; from offset (indirect) */
 
 /* Second bytes of opcodes with a ModRM */
 #define X86_OPS_2BYTE_MRM(X) \
-	X(X86_2B_NOP,      0x0D) /* Variable length NOP (3-9 with prefix) */ \
-	X(X86_2B_HINTS1,   0x18) /* Prefetch and hint-nop block 1/8 */ \
-	X(X86_2B_HINTS2,   0x19) /* Prefetch and hint-nop block 2/8 */ \
-	X(X86_2B_HINTS3,   0x1A) /* Prefetch and hint-nop block 3/8 */ \
-	X(X86_2B_HINTS4,   0x1B) /* Prefetch and hint-nop block 4/8 */ \
-	X(X86_2B_HINTS5,   0x1C) /* Prefetch and hint-nop block 5/8 */ \
-	X(X86_2B_HINTS6,   0x1D) /* Prefetch and hint-nop block 6/8 */ \
-	X(X86_2B_HINTS7,   0x1E) /* Prefetch and hint-nop block 7/8 */ \
-	X(X86_2B_HINTS8,   0x1F) /* Prefetch and hint-nop block 8/8 */ \
-	X(X86_2B_CMOVO,    0x40) \
-	X(X86_2B_CMOVNO,   0x41) \
-	X(X86_2B_CMOVB,    0x42) /* AKA CMOVC */ \
-	X(X86_2B_CMOVNB,   0x43) /* AKA CMOVNC */ \
-	X(X86_2B_CMOVZ,    0x44) /* AKA CMOVE */ \
-	X(X86_2B_CMOVNZ,   0x45) /* AKA CMOVNE */ \
-	X(X86_2B_CMOVNA,   0x46) /* AKA CMOVBE */ \
-	X(X86_2B_CMOVA,    0x47) /* AKA CMOVNBE */ \
-	X(X86_2B_CMOVS,    0x48) \
-	X(X86_2B_CMOVNS,   0x49) \
-	X(X86_2B_CMOVP,    0x4A) \
-	X(X86_2B_CMOVNP,   0x4B) \
-	X(X86_2B_CMOVL,    0x4C) /* AKA CMOVNGE */ \
-	X(X86_2B_CMOVNL,   0x4D) /* AKA CMOVGE */ \
-	X(X86_2B_CMOVNG,   0x4E) /* AKA CMOVLE */ \
-	X(X86_2B_CMOVG,    0x4F) /* AKA CMOVNLE */ \
-	X(X86_2B_SETO,     0x90) \
-	X(X86_2B_SETNO,    0x91) \
-	X(X86_2B_SETB,     0x92) /* AKA SETC */ \
-	X(X86_2B_SETNB,    0x93) /* AKA SETNC */ \
-	X(X86_2B_SETZ,     0x94) /* AKA SETE */ \
-	X(X86_2B_SETNZ,    0x95) /* AKA SETNZ */ \
-	X(X86_2B_SETNA,    0x96) /* AKA SETBE */ \
-	X(X86_2B_SETA,     0x97) /* AKA SETNBE */ \
-	X(X86_2B_SETS,     0x98) \
-	X(X86_2B_SETNS,    0x99) \
-	X(X86_2B_SETP,     0x9A) \
-	X(X86_2B_SETNP,    0x9B) \
-	X(X86_2B_SETL,     0x9C) /* AKA SETNGE */ \
-	X(X86_2B_SETNL,    0x9D) /* AKA SETGE */ \
-	X(X86_2B_SETNG,    0x9E) /* AKA SETLE */ \
-	X(X86_2B_SETG,     0x9F) /* AKA SETNLE */ \
-	X(X86_2B_BTMR,     0xA3) \
-	X(X86_2B_SHLDMRCL, 0xA5) \
-	X(X86_2B_BTS,      0xAB) \
-	X(X86_2B_SHRDMRCL, 0xAD) \
-	X(X86_2B_MISC,     0xAE) /* Float env stuff, memory fences */ \
-	X(X86_2B_IMUL,     0xAF) \
-	X(X86_2B_CMPXCHG8, 0xB0) \
-	X(X86_2B_CMPXCHGW, 0xB1) \
-	X(X86_2B_MOVZX8,   0xB6) \
-	X(X86_2B_MOVZXW,   0xB7) \
-	X(X86_2B_POPCNT,   0xB8) \
-	X(X86_2B_BTCRM,    0xBB) \
-	X(X86_2B_BSF,      0xBC) \
-	X(X86_2B_BSR,      0xBD) \
-	X(X86_2B_MOVSX8,   0xBE) \
-	X(X86_2B_MOVSXW,   0xBF) \
-	X(X86_2B_XADDRM8,  0xC0) \
-	X(X86_2B_XADDRMW,  0xC1) \
+	X(X86_2B_NOP,        0x0D) /* Variable length NOP (3-9 with prefix) */ \
+	X(X86_2B_HINTS1,     0x18) /* Prefetch and hint-nop block 1/8 */ \
+	X(X86_2B_HINTS2,     0x19) /* Prefetch and hint-nop block 2/8 */ \
+	X(X86_2B_HINTS3,     0x1A) /* Prefetch and hint-nop block 3/8 */ \
+	X(X86_2B_HINTS4,     0x1B) /* Prefetch and hint-nop block 4/8 */ \
+	X(X86_2B_HINTS5,     0x1C) /* Prefetch and hint-nop block 5/8 */ \
+	X(X86_2B_HINTS6,     0x1D) /* Prefetch and hint-nop block 6/8 */ \
+	X(X86_2B_HINTS7,     0x1E) /* Prefetch and hint-nop block 7/8 */ \
+	X(X86_2B_HINTS8,     0x1F) /* Prefetch and hint-nop block 8/8 */ \
+	X(X86_2B_CMOVO,      0x40) \
+	X(X86_2B_CMOVNO,     0x41) \
+	X(X86_2B_CMOVB,      0x42) /* AKA CMOVC */ \
+	X(X86_2B_CMOVNB,     0x43) /* AKA CMOVNC */ \
+	X(X86_2B_CMOVZ,      0x44) /* AKA CMOVE */ \
+	X(X86_2B_CMOVNZ,     0x45) /* AKA CMOVNE */ \
+	X(X86_2B_CMOVNA,     0x46) /* AKA CMOVBE */ \
+	X(X86_2B_CMOVA,      0x47) /* AKA CMOVNBE */ \
+	X(X86_2B_CMOVS,      0x48) \
+	X(X86_2B_CMOVNS,     0x49) \
+	X(X86_2B_CMOVP,      0x4A) \
+	X(X86_2B_CMOVNP,     0x4B) \
+	X(X86_2B_CMOVL,      0x4C) /* AKA CMOVNGE */ \
+	X(X86_2B_CMOVNL,     0x4D) /* AKA CMOVGE */ \
+	X(X86_2B_CMOVNG,     0x4E) /* AKA CMOVLE */ \
+	X(X86_2B_CMOVG,      0x4F) /* AKA CMOVNLE */ \
+	X(X86_2B_SETO,       0x90) \
+	X(X86_2B_SETNO,      0x91) \
+	X(X86_2B_SETB,       0x92) /* AKA SETC */ \
+	X(X86_2B_SETNB,      0x93) /* AKA SETNC */ \
+	X(X86_2B_SETZ,       0x94) /* AKA SETE */ \
+	X(X86_2B_SETNZ,      0x95) /* AKA SETNZ */ \
+	X(X86_2B_SETNA,      0x96) /* AKA SETBE */ \
+	X(X86_2B_SETA,       0x97) /* AKA SETNBE */ \
+	X(X86_2B_SETS,       0x98) \
+	X(X86_2B_SETNS,      0x99) \
+	X(X86_2B_SETP,       0x9A) \
+	X(X86_2B_SETNP,      0x9B) \
+	X(X86_2B_SETL,       0x9C) /* AKA SETNGE */ \
+	X(X86_2B_SETNL,      0x9D) /* AKA SETGE */ \
+	X(X86_2B_SETNG,      0x9E) /* AKA SETLE */ \
+	X(X86_2B_SETG,       0x9F) /* AKA SETNLE */ \
+	X(X86_2B_BTMR,       0xA3) \
+	X(X86_2B_SHLDMRCL,   0xA5) \
+	X(X86_2B_BTS,        0xAB) \
+	X(X86_2B_SHRDMRCL,   0xAD) \
+	X(X86_2B_MISC,       0xAE) /* Float env stuff, memory fences */ \
+	X(X86_2B_IMUL,       0xAF) \
+	X(X86_2B_CMPXCHG8,   0xB0) \
+	X(X86_2B_CMPXCHGW,   0xB1) \
+	X(X86_2B_MOVZX8,     0xB6) \
+	X(X86_2B_MOVZXW,     0xB7) \
+	X(X86_2B_POPCNT,     0xB8) \
+	X(X86_2B_BTCRM,      0xBB) \
+	X(X86_2B_BSF,        0xBC) \
+	X(X86_2B_BSR,        0xBD) \
+	X(X86_2B_MOVSX8,     0xBE) \
+	X(X86_2B_MOVSXW,     0xBF) \
+	X(X86_2B_XADDRM8,    0xC0) \
+	X(X86_2B_XADDRMW,    0xC1) \
 	/* NOTE: this one is actually a block with some VMX stuff too; it's only
 	   CMPXCHG64 (CMPXCHG8B if you prefer) if MRM.reg = 1, but naming it this
 	   way seemed more useful since it's what you'll see in normal userspace
 	   programs, which is what we're interested in. */ \
-	X(X86_2B_CMPXCHG64, 0xC7)
+	X(X86_2B_CMPXCHG64,  0xC7) \
+	/* -- MMX/SSE1/SSE2 instructions -- */ \
+	/* XXX: some of the naming here isn't totally perfect */ \
+	X(X86_2B_MOVRM128,   0x10) /* MOVUPS/MOVSS/MOVUPD/MOVD via prefix */ \
+	X(X86_2B_MOVMR128,   0x11) /* MOVUPS/MOVSS/MOVUPD/MOVD via prefix */ \
+	X(X86_2B_MOVLRM,     0x12) /* MOV(H)LPS/MOVLPD/MOVDDUP/MOVSLDUP */ \
+	X(X86_2B_MOVLMR,     0x13) /* MOVLP{S,D} */ \
+	X(X86_2B_UNPCKL,     0x14) /* UNPCKLP{S,D} */ \
+	X(X86_2B_UNPCKH,     0x15) /* UNPCKHPS/UNPCKHPD */ \
+	X(X86_2B_MOVHRM,     0x16) /* MOV(L)HPS/MOVHPD/MOVSHDUP */ \
+	X(X86_2B_MOVHMR,     0x17) /* MOVHPS/MOVHPD */ \
+	X(X86_2B_MOVARM,     0x28) /* MOVAP{S,D} via prefix */ \
+	X(X86_2B_MOVAMR,     0x29) /* MOVAP{S,D} */ \
+	X(X86_2B_CVTIF64,    0x2A) /* CVTxI2x{S,D} */ \
+	X(X86_2B_MOVNT,      0x2B) /* MOVNTP{S,D} */ \
+	X(X86_2B_CVTFT64,    0x2C) /* CVTTx{S,D}2xI */ \
+	X(X86_2B_CVTFI64,    0x2D) /* CVTx{S,D}2xI */ \
+	X(X86_2B_UCOMI,      0x2E) /* UCOMIS{S,D} */ \
+	X(X86_2B_COMI,       0x2F) /* COMIS{S,D} */ \
+	X(X86_2B_MOVMSK,     0x50) /* MOVMSDKP{S,D} */ \
+	X(X86_2B_SQRT,       0x51) /* SQRT{P,S}{S,D} */ \
+	X(X86_2B_RSQRT,      0x52) /* RSQRT{P,S}{S,D} */ \
+	X(X86_2B_RCP,        0x53) /* RCP{P,S}S */ \
+	X(X86_2B_AND,        0x54) /* ANDP{S,D} */ \
+	X(X86_2B_ANDN,       0x55) /* ANDNP{S,D} */ \
+	X(X86_2B_OR,         0x56) /* ORP{S,D} */ \
+	X(X86_2B_XOR,        0x57) /* XORP{S,D} */ \
+	X(X86_2B_ADD,        0x58) /* ADD{P,S}{S,D} */ \
+	X(X86_2B_MUL,        0x59) /* MUL{P,S}{S,D} */ \
+	X(X86_2B_CVTFF128,   0x5A) /* CVTxS2xD/CVTxS2xS */ \
+	X(X86_2B_CVTFI128,   0x5B) /* CVTDQ2PS/CVTPS2DQ/CVTTPS2DQ */ \
+	X(X86_2B_SUB,        0x5C) /* SUB{P,S}{S,D} */ \
+	X(X86_2B_DIV,        0x5D) /* DIV{P,S}{S,D} */ \
+	X(X86_2B_MIN,        0x5E) /* MIN{P,S}{S,D} */ \
+	X(X86_2B_MAX,        0x5F) /* MAX{P,S}{S,D} */ \
+	X(X86_2B_PUNPCKLBW,  0x60) \
+	X(X86_2B_PUNPCKLBD,  0x61) \
+	X(X86_2B_PUNPCKLDQ,  0x62) \
+	X(X86_2B_PACKSSWB,   0x63) \
+	X(X86_2B_PCMPGTB,    0x64) \
+	X(X86_2B_PCMPGTW,    0x65) \
+	X(X86_2B_PCMPGTD,    0x66) \
+	X(X86_2B_PACKUSWB,   0x67) \
+	X(X86_2B_PUNPCKHBW,  0x68) \
+	X(X86_2B_PUNPCKHWD,  0x69) \
+	X(X86_2B_PUNPCKHDQ,  0x6A) \
+	X(X86_2B_PACKSSDW,   0x6B) \
+	X(X86_2B_PUNPCKLQDQ, 0x6C) \
+	X(X86_2B_PUNPCKHQDQ, 0x6D) \
+	X(X86_2B_MOVDRM,     0x6E) \
+	X(X86_2B_MOVQRM,     0x6F) /* MOVQ/MOVDQA/MOVDQU */ \
+	X(X86_2B_PCMPEQB,    0x74) \
+	X(X86_2B_PCMPEQW,    0x75) \
+	X(X86_2B_PCMPEQD,    0x76) \
+	X(X86_2B_MOVDMR,     0x7E) \
+	X(X86_2B_MOVQMR,     0x7F) \
+	X(X86_2B_MOVNTI,     0xC3) \
+	X(X86_2B_ADDSUB,     0xD0) /* ADDSUBP{S,D} */ \
+	X(X86_2B_PSRLW,      0xD1) \
+	X(X86_2B_PSRLD,      0xD2) \
+	X(X86_2B_PSRLQ,      0xD3) \
+	X(X86_2B_PADDQ,      0xD4) \
+	X(X86_2B_PMULLW,     0xD5) \
+	X(X86_2B_MOVQRR,     0xD6) /* MOVQ(m,r)/MOVQ2DQ/MOVQ2DQ based on prefix */ \
+	X(X86_2B_PMOVMSKB,   0xD7) /* MOVQ2DQ/MOVDQ2Q */ \
+	X(X86_2B_PSUBUSB,    0xD8) \
+	X(X86_2B_PSUBUSW,    0xD9) \
+	X(X86_2B_PMINUB,     0xDA) \
+	X(X86_2B_PAND,       0xDB) \
+	X(X86_2B_PADDUSB,    0xDC) \
+	X(X86_2B_PADDUSW,    0xDD) \
+	X(X86_2B_PMAXUB,     0xDE) \
+	X(X86_2B_PANDN,      0xDF) \
+	X(X86_2B_PAVGB,      0xE0) \
+	X(X86_2B_PSRAW,      0xE1) \
+	X(X86_2B_PSRAD,      0xE2) \
+	X(X86_2B_PAVGW,      0xE3) \
+	X(X86_2B_PMULHUW,    0xE4) \
+	X(X86_2B_PMULHW,     0xE5) \
+	X(X86_2B_CVTQ,       0xE6) /* CVTPD2DQ/CVTTPD2DQ/CVTDQ2PD */ \
+	X(X86_2B_MOVNTQ,     0xE7) \
+	X(X86_2B_PSUBSB,     0xE8) \
+	X(X86_2B_PSUBSW,     0xE9) \
+	X(X86_2B_PMINSB,     0xEA) \
+	X(X86_2B_PMINSW,     0xEB) \
+	X(X86_2B_PADDSB,     0xEC) \
+	X(X86_2B_PADDSW,     0xED) \
+	X(X86_2B_PMAXSW,     0xEE) \
+	X(X86_2B_PXOR,       0xEF) \
+	X(X86_2B_LDDQU,      0xF0) \
+	X(X86_2B_PSLLW,      0xF1) \
+	X(X86_2B_PSLLD,      0xF2) \
+	X(X86_2B_PSLLQ,      0xF3) \
+	X(X86_2B_PMULUDQ,    0xF4) \
+	X(X86_2B_PMADDWD,    0xF5) \
+	X(X86_2B_PSABDW,     0xF6) \
+	X(X86_2B_MASKMOVQ,   0xF7) \
+	X(X86_2B_PSUBB,      0xF8) \
+	X(X86_2B_PSUBW,      0xF9) \
+	X(X86_2B_PSUBD,      0xFA) \
+	X(X86_2B_PSUBQ,      0xFB) \
+	X(X86_2B_PADDB,      0xFC) \
+	X(X86_2B_PADDW,      0xFD) \
+	X(X86_2B_PADDD,      0xFE)
 
 /* Second bytes of opcodes with a ModRM and a 1-byte immediate operand */
 #define X86_OPS_2BYTE_MRM_I8(X) \
 	X(X86_2B_SHLDMRI, 0xA4) \
 	X(X86_2B_SHRDMRI, 0xAC) \
-	X(X86_2B_BTXMI,   0xBA) /* BT/BTS/BTR/BTC depending on MRM.reg (4-7) */
+	X(X86_2B_BTXMI,   0xBA) /* BT/BTS/BTR/BTC depending on MRM.reg (4-7) */ \
+	/* -- MMX/SSE1/SSE2 instructions -- */ \
+	X(X86_2B_PSHUF,   0x70) /* PSHUFW/PSHUFLW/PSHUFHW/PSHUFD via MRM.reg */ \
+	X(X86_2B_PSWI,    0x71) /* PSRLW/PSRAW/PSLLW via MRM.reg */ \
+	X(X86_2B_PSDI,    0x72) /* PSRLD/PSRAD/PSLLD via MRM.reg */ \
+	X(X86_2B_PSQI,    0x73) /* PSRLQ/PSRAQ/PSLLQ via MRM.reg */ \
+	X(X86_2B_CMPSI,   0xC2) /* CMP{P,S}{S,D} via prefix */ \
+	X(X86_2B_PINSRW,  0xC4) \
+	X(X86_2B_PEXTRW,  0xC5) \
+	X(X86_2B_SHUF,    0xC6) /* SHUFP{S,D} */ \
 
 #define X86_OPS_2BYTE(X) \
 	X86_OPS_2BYTE_NO(X) \