Please file new bugs on Launchpad: Invirt or XVM (if you're not sure which, just pick one)

Context Navigation

source: trunk/packages/xen-3.1/xen-3.1/xen/arch/x86/x86_emulate.c @ 34

Last change on this file since 34 was 34, checked in by hartmans, 17 years ago
Add xen and xen-common
File size: 84.2 KB

Rev	Line
[34]	1	/******************************************************************************
	2	* x86_emulate.c
	3	*
	4	* Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
	5	*
	6	* Copyright (c) 2005-2007 Keir Fraser
	7	* Copyright (c) 2005-2007 XenSource Inc.
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU General Public License
	20	* along with this program; if not, write to the Free Software
	21	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	22	*/
	23
	24	#ifndef __XEN__
	25	#include <stddef.h>
	26	#include <stdint.h>
	27	#include <public/xen.h>
	28	#else
	29	#include <xen/config.h>
	30	#include <xen/types.h>
	31	#include <xen/lib.h>
	32	#include <asm/regs.h>
	33	#undef cmpxchg
	34	#endif
	35	#include <asm-x86/x86_emulate.h>
	36
	37	/* Operand sizes: 8-bit operands or specified/overridden size. */
	38	#define ByteOp (1<<0) /* 8-bit operands. */
	39	/* Destination operand type. */
	40	#define DstBitBase (0<<1) /* Memory operand, bit string. */
	41	#define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
	42	#define DstReg (2<<1) /* Register operand. */
	43	#define DstMem (3<<1) /* Memory operand. */
	44	#define DstMask (3<<1)
	45	/* Source operand type. */
	46	#define SrcNone (0<<3) /* No source operand. */
	47	#define SrcImplicit (0<<3) /* Source operand is implicit in the opcode. */
	48	#define SrcReg (1<<3) /* Register operand. */
	49	#define SrcMem (2<<3) /* Memory operand. */
	50	#define SrcMem16 (3<<3) /* Memory operand (16-bit). */
	51	#define SrcImm (4<<3) /* Immediate operand. */
	52	#define SrcImmByte (5<<3) /* 8-bit sign-extended immediate operand. */
	53	#define SrcMask (7<<3)
	54	/* Generic ModRM decode. */
	55	#define ModRM (1<<6)
	56	/* Destination is only written; never read. */
	57	#define Mov (1<<7)
	58
	59	static uint8_t opcode_table[256] = {
	60	/* 0x00 - 0x07 */
	61	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	62	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	63	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, 0,
	64	/* 0x08 - 0x0F */
	65	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	66	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	67	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, 0,
	68	/* 0x10 - 0x17 */
	69	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	70	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	71	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, 0,
	72	/* 0x18 - 0x1F */
	73	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	74	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	75	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, 0,
	76	/* 0x20 - 0x27 */
	77	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	78	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	79	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, ImplicitOps,
	80	/* 0x28 - 0x2F */
	81	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	82	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	83	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, ImplicitOps,
	84	/* 0x30 - 0x37 */
	85	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	86	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	87	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, ImplicitOps,
	88	/* 0x38 - 0x3F */
	89	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	90	ByteOp\|DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	91	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm, 0, ImplicitOps,
	92	/* 0x40 - 0x4F */
	93	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	94	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	95	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	96	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	97	/* 0x50 - 0x5F */
	98	ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov,
	99	ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov,
	100	ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov,
	101	ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov, ImplicitOps\|Mov,
	102	/* 0x60 - 0x67 */
	103	ImplicitOps, ImplicitOps, DstReg\|SrcMem\|ModRM, DstReg\|SrcMem16\|ModRM\|Mov,
	104	0, 0, 0, 0,
	105	/* 0x68 - 0x6F */
	106	ImplicitOps\|Mov, DstMem\|SrcImm\|ModRM\|Mov,
	107	ImplicitOps\|Mov, DstMem\|SrcImmByte\|ModRM\|Mov,
	108	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	109	/* 0x70 - 0x77 */
	110	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	111	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	112	/* 0x78 - 0x7F */
	113	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	114	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	115	/* 0x80 - 0x87 */
	116	ByteOp\|DstMem\|SrcImm\|ModRM, DstMem\|SrcImm\|ModRM,
	117	ByteOp\|DstMem\|SrcImm\|ModRM, DstMem\|SrcImmByte\|ModRM,
	118	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	119	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	120	/* 0x88 - 0x8F */
	121	ByteOp\|DstMem\|SrcReg\|ModRM\|Mov, DstMem\|SrcReg\|ModRM\|Mov,
	122	ByteOp\|DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	123	0, DstReg\|SrcNone\|ModRM, 0, DstMem\|SrcNone\|ModRM\|Mov,
	124	/* 0x90 - 0x97 */
	125	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	126	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	127	/* 0x98 - 0x9F */
	128	ImplicitOps, ImplicitOps, 0, 0, 0, 0, ImplicitOps, ImplicitOps,
	129	/* 0xA0 - 0xA7 */
	130	ByteOp\|ImplicitOps\|Mov, ImplicitOps\|Mov,
	131	ByteOp\|ImplicitOps\|Mov, ImplicitOps\|Mov,
	132	ByteOp\|ImplicitOps\|Mov, ImplicitOps\|Mov, 0, 0,
	133	/* 0xA8 - 0xAF */
	134	ByteOp\|DstReg\|SrcImm, DstReg\|SrcImm,
	135	ByteOp\|ImplicitOps\|Mov, ImplicitOps\|Mov,
	136	ByteOp\|ImplicitOps\|Mov, ImplicitOps\|Mov, 0, 0,
	137	/* 0xB0 - 0xB7 */
	138	ByteOp\|DstReg\|SrcImm\|Mov, ByteOp\|DstReg\|SrcImm\|Mov,
	139	ByteOp\|DstReg\|SrcImm\|Mov, ByteOp\|DstReg\|SrcImm\|Mov,
	140	ByteOp\|DstReg\|SrcImm\|Mov, ByteOp\|DstReg\|SrcImm\|Mov,
	141	ByteOp\|DstReg\|SrcImm\|Mov, ByteOp\|DstReg\|SrcImm\|Mov,
	142	/* 0xB8 - 0xBF */
	143	DstReg\|SrcImm\|Mov, DstReg\|SrcImm\|Mov, DstReg\|SrcImm\|Mov, DstReg\|SrcImm\|Mov,
	144	DstReg\|SrcImm\|Mov, DstReg\|SrcImm\|Mov, DstReg\|SrcImm\|Mov, DstReg\|SrcImm\|Mov,
	145	/* 0xC0 - 0xC7 */
	146	ByteOp\|DstMem\|SrcImm\|ModRM, DstMem\|SrcImmByte\|ModRM,
	147	ImplicitOps, ImplicitOps,
	148	0, 0, ByteOp\|DstMem\|SrcImm\|ModRM\|Mov, DstMem\|SrcImm\|ModRM\|Mov,
	149	/* 0xC8 - 0xCF */
	150	0, 0, 0, 0, 0, 0, 0, 0,
	151	/* 0xD0 - 0xD7 */
	152	ByteOp\|DstMem\|SrcImplicit\|ModRM, DstMem\|SrcImplicit\|ModRM,
	153	ByteOp\|DstMem\|SrcImplicit\|ModRM, DstMem\|SrcImplicit\|ModRM,
	154	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	155	/* 0xD8 - 0xDF */
	156	0, 0, 0, 0, 0, 0, 0, 0,
	157	/* 0xE0 - 0xE7 */
	158	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	159	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	160	/* 0xE8 - 0xEF */
	161	ImplicitOps, ImplicitOps, 0, ImplicitOps,
	162	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	163	/* 0xF0 - 0xF7 */
	164	0, 0, 0, 0,
	165	0, ImplicitOps, ByteOp\|DstMem\|SrcNone\|ModRM, DstMem\|SrcNone\|ModRM,
	166	/* 0xF8 - 0xFF */
	167	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	168	ImplicitOps, ImplicitOps, ByteOp\|DstMem\|SrcNone\|ModRM, DstMem\|SrcNone\|ModRM
	169	};
	170
	171	static uint8_t twobyte_table[256] = {
	172	/* 0x00 - 0x07 */
	173	0, 0, 0, 0, 0, ImplicitOps, 0, 0,
	174	/* 0x08 - 0x0F */
	175	ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps\|ModRM, 0, 0,
	176	/* 0x10 - 0x17 */
	177	0, 0, 0, 0, 0, 0, 0, 0,
	178	/* 0x18 - 0x1F */
	179	ImplicitOps\|ModRM, ImplicitOps\|ModRM, ImplicitOps\|ModRM, ImplicitOps\|ModRM,
	180	ImplicitOps\|ModRM, ImplicitOps\|ModRM, ImplicitOps\|ModRM, ImplicitOps\|ModRM,
	181	/* 0x20 - 0x27 */
	182	ImplicitOps\|ModRM, ImplicitOps\|ModRM, ImplicitOps\|ModRM, ImplicitOps\|ModRM,
	183	0, 0, 0, 0,
	184	/* 0x28 - 0x2F */
	185	0, 0, 0, 0, 0, 0, 0, 0,
	186	/* 0x30 - 0x37 */
	187	ImplicitOps, 0, ImplicitOps, 0, 0, 0, 0, 0,
	188	/* 0x38 - 0x3F */
	189	0, 0, 0, 0, 0, 0, 0, 0,
	190	/* 0x40 - 0x47 */
	191	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	192	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	193	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	194	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	195	/* 0x48 - 0x4F */
	196	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	197	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	198	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	199	DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem\|ModRM\|Mov,
	200	/* 0x50 - 0x5F */
	201	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	202	/* 0x60 - 0x6F */
	203	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	204	/* 0x70 - 0x7F */
	205	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	206	/* 0x80 - 0x87 */
	207	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	208	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	209	/* 0x88 - 0x8F */
	210	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	211	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	212	/* 0x90 - 0x97 */
	213	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	214	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	215	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	216	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	217	/* 0x98 - 0x9F */
	218	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	219	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	220	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	221	ByteOp\|DstMem\|SrcNone\|ModRM\|Mov, ByteOp\|DstMem\|SrcNone\|ModRM\|Mov,
	222	/* 0xA0 - 0xA7 */
	223	0, 0, 0, DstBitBase\|SrcReg\|ModRM, 0, 0, 0, 0,
	224	/* 0xA8 - 0xAF */
	225	0, 0, 0, DstBitBase\|SrcReg\|ModRM, 0, 0, 0, DstReg\|SrcMem\|ModRM,
	226	/* 0xB0 - 0xB7 */
	227	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM,
	228	0, DstBitBase\|SrcReg\|ModRM,
	229	0, 0, ByteOp\|DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem16\|ModRM\|Mov,
	230	/* 0xB8 - 0xBF */
	231	0, 0, DstBitBase\|SrcImmByte\|ModRM, DstBitBase\|SrcReg\|ModRM,
	232	DstReg\|SrcMem\|ModRM, DstReg\|SrcMem\|ModRM,
	233	ByteOp\|DstReg\|SrcMem\|ModRM\|Mov, DstReg\|SrcMem16\|ModRM\|Mov,
	234	/* 0xC0 - 0xC7 */
	235	ByteOp\|DstMem\|SrcReg\|ModRM, DstMem\|SrcReg\|ModRM, 0, 0,
	236	0, 0, 0, ImplicitOps\|ModRM,
	237	/* 0xC8 - 0xCF */
	238	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	239	ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps,
	240	/* 0xD0 - 0xDF */
	241	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	242	/* 0xE0 - 0xEF */
	243	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	244	/* 0xF0 - 0xFF */
	245	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	246	};
	247
	248	/* Type, address-of, and value of an instruction's operand. */
	249	struct operand {
	250	enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type;
	251	unsigned int bytes;
	252	unsigned long val, orig_val;
	253	union {
	254	/* OP_REG: Pointer to register field. */
	255	unsigned long *reg;
	256	/* OP_MEM: Segment and offset. */
	257	struct {
	258	enum x86_segment seg;
	259	unsigned long off;
	260	} mem;
	261	};
	262	};
	263
	264	/* EFLAGS bit definitions. */
	265	#define EFLG_OF (1<<11)
	266	#define EFLG_DF (1<<10)
	267	#define EFLG_IF (1<<9)
	268	#define EFLG_SF (1<<7)
	269	#define EFLG_ZF (1<<6)
	270	#define EFLG_AF (1<<4)
	271	#define EFLG_PF (1<<2)
	272	#define EFLG_CF (1<<0)
	273
	274	/* Exception definitions. */
	275	#define EXC_DE 0
	276	#define EXC_BR 5
	277	#define EXC_UD 6
	278	#define EXC_GP 13
	279
	280	/*
	281	* Instruction emulation:
	282	* Most instructions are emulated directly via a fragment of inline assembly
	283	* code. This allows us to save/restore EFLAGS and thus very easily pick up
	284	* any modified flags.
	285	*/
	286
	287	#if defined(__x86_64__)
	288	#define _LO32 "k" /* force 32-bit operand */
	289	#define _STK "%%rsp" /* stack pointer */
	290	#elif defined(__i386__)
	291	#define _LO32 "" /* force 32-bit operand */
	292	#define _STK "%%esp" /* stack pointer */
	293	#endif
	294
	295	/*
	296	* These EFLAGS bits are restored from saved value during emulation, and
	297	* any changes are written back to the saved value after emulation.
	298	*/
	299	#define EFLAGS_MASK (EFLG_OF\|EFLG_SF\|EFLG_ZF\|EFLG_AF\|EFLG_PF\|EFLG_CF)
	300
	301	/* Before executing instruction: restore necessary bits in EFLAGS. */
	302	#define _PRE_EFLAGS(_sav, _msk, _tmp) \
	303	/* EFLAGS = (_sav & _msk) \| (EFLAGS & ~_msk); */\
	304	"push %"_sav"; " \
	305	"movl %"_msk",%"_LO32 _tmp"; " \
	306	"andl %"_LO32 _tmp",("_STK"); " \
	307	"pushf; " \
	308	"notl %"_LO32 _tmp"; " \
	309	"andl %"_LO32 _tmp",("_STK"); " \
	310	"pop %"_tmp"; " \
	311	"orl %"_LO32 _tmp",("_STK"); " \
	312	"popf; " \
	313	/* _sav &= ~msk; */ \
	314	"movl %"_msk",%"_LO32 _tmp"; " \
	315	"notl %"_LO32 _tmp"; " \
	316	"andl %"_LO32 _tmp",%"_sav"; "
	317
	318	/* After executing instruction: write-back necessary bits in EFLAGS. */
	319	#define _POST_EFLAGS(_sav, _msk, _tmp) \
	320	/* _sav \|= EFLAGS & _msk; */ \
	321	"pushf; " \
	322	"pop %"_tmp"; " \
	323	"andl %"_msk",%"_LO32 _tmp"; " \
	324	"orl %"_LO32 _tmp",%"_sav"; "
	325
	326	/* Raw emulation: instruction has two explicit operands. */
	327	#define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy)\
	328	do{ unsigned long _tmp; \
	329	switch ( (_dst).bytes ) \
	330	{ \
	331	case 2: \
	332	__asm__ __volatile__ ( \
	333	_PRE_EFLAGS("0","4","2") \
	334	_op"w %"_wx"3,%1; " \
	335	_POST_EFLAGS("0","4","2") \
	336	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	337	: _wy ((_src).val), "i" (EFLAGS_MASK), \
	338	"m" (_eflags), "m" ((_dst).val) ); \
	339	break; \
	340	case 4: \
	341	__asm__ __volatile__ ( \
	342	_PRE_EFLAGS("0","4","2") \
	343	_op"l %"_lx"3,%1; " \
	344	_POST_EFLAGS("0","4","2") \
	345	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	346	: _ly ((_src).val), "i" (EFLAGS_MASK), \
	347	"m" (_eflags), "m" ((_dst).val) ); \
	348	break; \
	349	case 8: \
	350	__emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy); \
	351	break; \
	352	} \
	353	} while (0)
	354	#define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)\
	355	do{ unsigned long _tmp; \
	356	switch ( (_dst).bytes ) \
	357	{ \
	358	case 1: \
	359	__asm__ __volatile__ ( \
	360	_PRE_EFLAGS("0","4","2") \
	361	_op"b %"_bx"3,%1; " \
	362	_POST_EFLAGS("0","4","2") \
	363	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	364	: _by ((_src).val), "i" (EFLAGS_MASK), \
	365	"m" (_eflags), "m" ((_dst).val) ); \
	366	break; \
	367	default: \
	368	__emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy);\
	369	break; \
	370	} \
	371	} while (0)
	372	/* Source operand is byte-sized and may be restricted to just %cl. */
	373	#define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
	374	__emulate_2op(_op, _src, _dst, _eflags, \
	375	"b", "c", "b", "c", "b", "c", "b", "c")
	376	/* Source operand is byte, word, long or quad sized. */
	377	#define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
	378	__emulate_2op(_op, _src, _dst, _eflags, \
	379	"b", "q", "w", "r", _LO32, "r", "", "r")
	380	/* Source operand is word, long or quad sized. */
	381	#define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
	382	__emulate_2op_nobyte(_op, _src, _dst, _eflags, \
	383	"w", "r", _LO32, "r", "", "r")
	384
	385	/* Instruction has only one explicit operand (no source operand). */
	386	#define emulate_1op(_op,_dst,_eflags) \
	387	do{ unsigned long _tmp; \
	388	switch ( (_dst).bytes ) \
	389	{ \
	390	case 1: \
	391	__asm__ __volatile__ ( \
	392	_PRE_EFLAGS("0","3","2") \
	393	_op"b %1; " \
	394	_POST_EFLAGS("0","3","2") \
	395	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	396	: "i" (EFLAGS_MASK), "m" (_eflags), "m" ((_dst).val) ); \
	397	break; \
	398	case 2: \
	399	__asm__ __volatile__ ( \
	400	_PRE_EFLAGS("0","3","2") \
	401	_op"w %1; " \
	402	_POST_EFLAGS("0","3","2") \
	403	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	404	: "i" (EFLAGS_MASK), "m" (_eflags), "m" ((_dst).val) ); \
	405	break; \
	406	case 4: \
	407	__asm__ __volatile__ ( \
	408	_PRE_EFLAGS("0","3","2") \
	409	_op"l %1; " \
	410	_POST_EFLAGS("0","3","2") \
	411	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	412	: "i" (EFLAGS_MASK), "m" (_eflags), "m" ((_dst).val) ); \
	413	break; \
	414	case 8: \
	415	__emulate_1op_8byte(_op, _dst, _eflags); \
	416	break; \
	417	} \
	418	} while (0)
	419
	420	/* Emulate an instruction with quadword operands (x86/64 only). */
	421	#if defined(__x86_64__)
	422	#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) \
	423	do{ __asm__ __volatile__ ( \
	424	_PRE_EFLAGS("0","4","2") \
	425	_op"q %"_qx"3,%1; " \
	426	_POST_EFLAGS("0","4","2") \
	427	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	428	: _qy ((_src).val), "i" (EFLAGS_MASK), \
	429	"m" (_eflags), "m" ((_dst).val) ); \
	430	} while (0)
	431	#define __emulate_1op_8byte(_op, _dst, _eflags) \
	432	do{ __asm__ __volatile__ ( \
	433	_PRE_EFLAGS("0","3","2") \
	434	_op"q %1; " \
	435	_POST_EFLAGS("0","3","2") \
	436	: "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \
	437	: "i" (EFLAGS_MASK), "m" (_eflags), "m" ((_dst).val) ); \
	438	} while (0)
	439	#elif defined(__i386__)
	440	#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy)
	441	#define __emulate_1op_8byte(_op, _dst, _eflags)
	442	#endif /* __i386__ */
	443
	444	/* Fetch next part of the instruction being emulated. */
	445	#define insn_fetch_bytes(_size) \
	446	({ unsigned long _x, _eip = _regs.eip; \
	447	if ( !mode_64bit() ) _eip = (uint32_t)_eip; /* ignore upper dword */ \
	448	_regs.eip += (_size); /* real hardware doesn't truncate */ \
	449	generate_exception_if((uint8_t)(_regs.eip - ctxt->regs->eip) > 15, \
	450	EXC_GP); \
	451	rc = ops->insn_fetch(x86_seg_cs, _eip, &_x, (_size), ctxt); \
	452	if ( rc ) goto done; \
	453	_x; \
	454	})
	455	#define insn_fetch_type(_type) ((_type)insn_fetch_bytes(sizeof(_type)))
	456
	457	#define _truncate_ea(ea, byte_width) \
	458	({ unsigned long __ea = (ea); \
	459	unsigned int _width = (byte_width); \
	460	((_width == sizeof(unsigned long)) ? __ea : \
	461	(__ea & ((1UL << (_width << 3)) - 1))); \
	462	})
	463	#define truncate_ea(ea) _truncate_ea((ea), ad_bytes)
	464
	465	#define mode_64bit() (def_ad_bytes == 8)
	466
	467	#define fail_if(p) \
	468	do { \
	469	rc = (p) ? X86EMUL_UNHANDLEABLE : X86EMUL_OKAY; \
	470	if ( rc ) goto done; \
	471	} while (0)
	472
	473	/* In future we will be able to generate arbitrary exceptions. */
	474	#define generate_exception_if(p, e) fail_if(p)
	475
	476	/* To be done... */
	477	#define mode_ring0() (0)
	478	#define mode_iopl() (0)
	479
	480	/* Given byte has even parity (even number of 1s)? */
	481	static int even_parity(uint8_t v)
	482	{
	483	__asm__ ( "test %%al,%%al; setp %%al"
	484	: "=a" (v) : "0" (v) );
	485	return v;
	486	}
	487
	488	/* Update address held in a register, based on addressing mode. */
	489	#define _register_address_increment(reg, inc, byte_width) \
	490	do { \
	491	int _inc = (inc); /* signed type ensures sign extension to long */ \
	492	unsigned int _width = (byte_width); \
	493	if ( _width == sizeof(unsigned long) ) \
	494	(reg) += _inc; \
	495	else if ( mode_64bit() ) \
	496	(reg) = ((reg) + _inc) & ((1UL << (_width << 3)) - 1); \
	497	else \
	498	(reg) = ((reg) & ~((1UL << (_width << 3)) - 1)) \| \
	499	(((reg) + _inc) & ((1UL << (_width << 3)) - 1)); \
	500	} while (0)
	501	#define register_address_increment(reg, inc) \
	502	_register_address_increment((reg), (inc), ad_bytes)
	503
	504	#define sp_pre_dec(dec) ({ \
	505	_register_address_increment(_regs.esp, -(dec), ctxt->sp_size/8); \
	506	_truncate_ea(_regs.esp, ctxt->sp_size/8); \
	507	})
	508	#define sp_post_inc(inc) ({ \
	509	unsigned long __esp = _truncate_ea(_regs.esp, ctxt->sp_size/8); \
	510	_register_address_increment(_regs.esp, (inc), ctxt->sp_size/8); \
	511	__esp; \
	512	})
	513
	514	#define jmp_rel(rel) \
	515	do { \
	516	_regs.eip += (int)(rel); \
	517	if ( !mode_64bit() ) \
	518	_regs.eip = ((op_bytes == 2) \
	519	? (uint16_t)_regs.eip : (uint32_t)_regs.eip); \
	520	} while (0)
	521
	522	static int __handle_rep_prefix(
	523	struct cpu_user_regs *int_regs,
	524	struct cpu_user_regs *ext_regs,
	525	int ad_bytes)
	526	{
	527	unsigned long ecx = ((ad_bytes == 2) ? (uint16_t)int_regs->ecx :
	528	(ad_bytes == 4) ? (uint32_t)int_regs->ecx :
	529	int_regs->ecx);
	530
	531	if ( ecx-- == 0 )
	532	{
	533	ext_regs->eip = int_regs->eip;
	534	return 1;
	535	}
	536
	537	if ( ad_bytes == 2 )
	538	(uint16_t )&int_regs->ecx = ecx;
	539	else if ( ad_bytes == 4 )
	540	int_regs->ecx = (uint32_t)ecx;
	541	else
	542	int_regs->ecx = ecx;
	543	int_regs->eip = ext_regs->eip;
	544	return 0;
	545	}
	546
	547	#define handle_rep_prefix() \
	548	do { \
	549	if ( rep_prefix && __handle_rep_prefix(&_regs, ctxt->regs, ad_bytes) ) \
	550	goto done; \
	551	} while (0)
	552
	553	/*
	554	* Unsigned multiplication with double-word result.
	555	* IN: Multiplicand=m[0], Multiplier=m[1]
	556	* OUT: Return CF/OF (overflow status); Result=m[1]:m[0]
	557	*/
	558	static int mul_dbl(unsigned long m[2])
	559	{
	560	int rc;
	561	asm ( "mul %4; seto %b2"
	562	: "=a" (m[0]), "=d" (m[1]), "=q" (rc)
	563	: "0" (m[0]), "1" (m[1]), "2" (0) );
	564	return rc;
	565	}
	566
	567	/*
	568	* Signed multiplication with double-word result.
	569	* IN: Multiplicand=m[0], Multiplier=m[1]
	570	* OUT: Return CF/OF (overflow status); Result=m[1]:m[0]
	571	*/
	572	static int imul_dbl(unsigned long m[2])
	573	{
	574	int rc;
	575	asm ( "imul %4; seto %b2"
	576	: "=a" (m[0]), "=d" (m[1]), "=q" (rc)
	577	: "0" (m[0]), "1" (m[1]), "2" (0) );
	578	return rc;
	579	}
	580
	581	/*
	582	* Unsigned division of double-word dividend.
	583	* IN: Dividend=u[1]:u[0], Divisor=v
	584	* OUT: Return 1: #DE
	585	* Return 0: Quotient=u[0], Remainder=u[1]
	586	*/
	587	static int div_dbl(unsigned long u[2], unsigned long v)
	588	{
	589	if ( (v == 0) \|\| (u[1] > v) \|\| ((u[1] == v) && (u[0] != 0)) )
	590	return 1;
	591	asm ( "div %4"
	592	: "=a" (u[0]), "=d" (u[1])
	593	: "0" (u[0]), "1" (u[1]), "r" (v) );
	594	return 0;
	595	}
	596
	597	/*
	598	* Signed division of double-word dividend.
	599	* IN: Dividend=u[1]:u[0], Divisor=v
	600	* OUT: Return 1: #DE
	601	* Return 0: Quotient=u[0], Remainder=u[1]
	602	* NB. We don't use idiv directly as it's moderately hard to work out
	603	* ahead of time whether it will #DE, which we cannot allow to happen.
	604	*/
	605	static int idiv_dbl(unsigned long u[2], unsigned long v)
	606	{
	607	int negu = (long)u[1] < 0, negv = (long)v < 0;
	608
	609	/* u = abs(u) */
	610	if ( negu )
	611	{
	612	u[1] = ~u[1];
	613	if ( (u[0] = -u[0]) == 0 )
	614	u[1]++;
	615	}
	616
	617	/* abs(u) / abs(v) */
	618	if ( div_dbl(u, negv ? -v : v) )
	619	return 1;
	620
	621	/* Remainder has same sign as dividend. It cannot overflow. */
	622	if ( negu )
	623	u[1] = -u[1];
	624
	625	/* Quotient is overflowed if sign bit is set. */
	626	if ( negu ^ negv )
	627	{
	628	if ( (long)u[0] >= 0 )
	629	u[0] = -u[0];
	630	else if ( (u[0] << 1) != 0 ) /* == 0x80...0 is okay */
	631	return 1;
	632	}
	633	else if ( (long)u[0] < 0 )
	634	return 1;
	635
	636	return 0;
	637	}
	638
	639	static int
	640	test_cc(
	641	unsigned int condition, unsigned int flags)
	642	{
	643	int rc = 0;
	644
	645	switch ( (condition & 15) >> 1 )
	646	{
	647	case 0: /* o */
	648	rc \|= (flags & EFLG_OF);
	649	break;
	650	case 1: /* b/c/nae */
	651	rc \|= (flags & EFLG_CF);
	652	break;
	653	case 2: /* z/e */
	654	rc \|= (flags & EFLG_ZF);
	655	break;
	656	case 3: /* be/na */
	657	rc \|= (flags & (EFLG_CF\|EFLG_ZF));
	658	break;
	659	case 4: /* s */
	660	rc \|= (flags & EFLG_SF);
	661	break;
	662	case 5: /* p/pe */
	663	rc \|= (flags & EFLG_PF);
	664	break;
	665	case 7: /* le/ng */
	666	rc \|= (flags & EFLG_ZF);
	667	/* fall through */
	668	case 6: /* l/nge */
	669	rc \|= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
	670	break;
	671	}
	672
	673	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	674	return (!!rc ^ (condition & 1));
	675	}
	676
	677	void *
	678	decode_register(
	679	uint8_t modrm_reg, struct cpu_user_regs *regs, int highbyte_regs)
	680	{
	681	void *p;
	682
	683	switch ( modrm_reg )
	684	{
	685	case 0: p = &regs->eax; break;
	686	case 1: p = &regs->ecx; break;
	687	case 2: p = &regs->edx; break;
	688	case 3: p = &regs->ebx; break;
	689	case 4: p = (highbyte_regs ?
	690	((unsigned char *)&regs->eax + 1) :
	691	(unsigned char *)&regs->esp); break;
	692	case 5: p = (highbyte_regs ?
	693	((unsigned char *)&regs->ecx + 1) :
	694	(unsigned char *)&regs->ebp); break;
	695	case 6: p = (highbyte_regs ?
	696	((unsigned char *)&regs->edx + 1) :
	697	(unsigned char *)&regs->esi); break;
	698	case 7: p = (highbyte_regs ?
	699	((unsigned char *)&regs->ebx + 1) :
	700	(unsigned char *)&regs->edi); break;
	701	#if defined(__x86_64__)
	702	case 8: p = &regs->r8; break;
	703	case 9: p = &regs->r9; break;
	704	case 10: p = &regs->r10; break;
	705	case 11: p = &regs->r11; break;
	706	case 12: p = &regs->r12; break;
	707	case 13: p = &regs->r13; break;
	708	case 14: p = &regs->r14; break;
	709	case 15: p = &regs->r15; break;
	710	#endif
	711	default: p = NULL; break;
	712	}
	713
	714	return p;
	715	}
	716
	717	int
	718	x86_emulate(
	719	struct x86_emulate_ctxt *ctxt,
	720	struct x86_emulate_ops *ops)
	721	{
	722	/* Shadow copy of register state. Committed on successful emulation. */
	723	struct cpu_user_regs _regs = *ctxt->regs;
	724
	725	uint8_t b, d, sib, sib_index, sib_base, twobyte = 0, rex_prefix = 0;
	726	uint8_t modrm, modrm_mod = 0, modrm_reg = 0, modrm_rm = 0;
	727	unsigned int op_bytes, def_op_bytes, ad_bytes, def_ad_bytes;
	728	unsigned int lock_prefix = 0, rep_prefix = 0;
	729	int override_seg = -1, rc = X86EMUL_OKAY;
	730	struct operand src, dst;
	731
	732	/* Data operand effective address (usually computed from ModRM). */
	733	struct operand ea;
	734
	735	/* Default is a memory operand relative to segment DS. */
	736	ea.type = OP_MEM;
	737	ea.mem.seg = x86_seg_ds;
	738	ea.mem.off = 0;
	739
	740	op_bytes = def_op_bytes = ad_bytes = def_ad_bytes = ctxt->addr_size/8;
	741	if ( op_bytes == 8 )
	742	{
	743	op_bytes = def_op_bytes = 4;
	744	#ifndef __x86_64__
	745	return X86EMUL_UNHANDLEABLE;
	746	#endif
	747	}
	748
	749	/* Prefix bytes. */
	750	for ( ; ; )
	751	{
	752	switch ( b = insn_fetch_type(uint8_t) )
	753	{
	754	case 0x66: /* operand-size override */
	755	op_bytes = def_op_bytes ^ 6;
	756	break;
	757	case 0x67: /* address-size override */
	758	ad_bytes = def_ad_bytes ^ (mode_64bit() ? 12 : 6);
	759	break;
	760	case 0x2e: /* CS override */
	761	override_seg = x86_seg_cs;
	762	break;
	763	case 0x3e: /* DS override */
	764	override_seg = x86_seg_ds;
	765	break;
	766	case 0x26: /* ES override */
	767	override_seg = x86_seg_es;
	768	break;
	769	case 0x64: /* FS override */
	770	override_seg = x86_seg_fs;
	771	break;
	772	case 0x65: /* GS override */
	773	override_seg = x86_seg_gs;
	774	break;
	775	case 0x36: /* SS override */
	776	override_seg = x86_seg_ss;
	777	break;
	778	case 0xf0: /* LOCK */
	779	lock_prefix = 1;
	780	break;
	781	case 0xf2: /* REPNE/REPNZ */
	782	case 0xf3: /* REP/REPE/REPZ */
	783	rep_prefix = 1;
	784	break;
	785	case 0x40 ... 0x4f: /* REX */
	786	if ( !mode_64bit() )
	787	goto done_prefixes;
	788	rex_prefix = b;
	789	continue;
	790	default:
	791	goto done_prefixes;
	792	}
	793
	794	/* Any legacy prefix after a REX prefix nullifies its effect. */
	795	rex_prefix = 0;
	796	}
	797	done_prefixes:
	798
	799	if ( rex_prefix & 8 ) /* REX.W */
	800	op_bytes = 8;
	801
	802	/* Opcode byte(s). */
	803	d = opcode_table[b];
	804	if ( d == 0 )
	805	{
	806	/* Two-byte opcode? */
	807	if ( b == 0x0f )
	808	{
	809	twobyte = 1;
	810	b = insn_fetch_type(uint8_t);
	811	d = twobyte_table[b];
	812	}
	813
	814	/* Unrecognised? */
	815	if ( d == 0 )
	816	goto cannot_emulate;
	817	}
	818
	819	/* Lock prefix is allowed only on RMW instructions. */
	820	generate_exception_if((d & Mov) && lock_prefix, EXC_GP);
	821
	822	/* ModRM and SIB bytes. */
	823	if ( d & ModRM )
	824	{
	825	modrm = insn_fetch_type(uint8_t);
	826	modrm_mod = (modrm & 0xc0) >> 6;
	827	modrm_reg = ((rex_prefix & 4) << 1) \| ((modrm & 0x38) >> 3);
	828	modrm_rm = modrm & 0x07;
	829
	830	if ( modrm_mod == 3 )
	831	{
	832	modrm_rm \|= (rex_prefix & 1) << 3;
	833	ea.type = OP_REG;
	834	ea.reg = decode_register(
	835	modrm_rm, &_regs, (d & ByteOp) && (rex_prefix == 0));
	836	}
	837	else if ( ad_bytes == 2 )
	838	{
	839	/* 16-bit ModR/M decode. */
	840	switch ( modrm_rm )
	841	{
	842	case 0:
	843	ea.mem.off = _regs.ebx + _regs.esi;
	844	break;
	845	case 1:
	846	ea.mem.off = _regs.ebx + _regs.edi;
	847	break;
	848	case 2:
	849	ea.mem.seg = x86_seg_ss;
	850	ea.mem.off = _regs.ebp + _regs.esi;
	851	break;
	852	case 3:
	853	ea.mem.seg = x86_seg_ss;
	854	ea.mem.off = _regs.ebp + _regs.edi;
	855	break;
	856	case 4:
	857	ea.mem.off = _regs.esi;
	858	break;
	859	case 5:
	860	ea.mem.off = _regs.edi;
	861	break;
	862	case 6:
	863	if ( modrm_mod == 0 )
	864	break;
	865	ea.mem.seg = x86_seg_ss;
	866	ea.mem.off = _regs.ebp;
	867	break;
	868	case 7:
	869	ea.mem.off = _regs.ebx;
	870	break;
	871	}
	872	switch ( modrm_mod )
	873	{
	874	case 0:
	875	if ( modrm_rm == 6 )
	876	ea.mem.off = insn_fetch_type(int16_t);
	877	break;
	878	case 1:
	879	ea.mem.off += insn_fetch_type(int8_t);
	880	break;
	881	case 2:
	882	ea.mem.off += insn_fetch_type(int16_t);
	883	break;
	884	}
	885	ea.mem.off = truncate_ea(ea.mem.off);
	886	}
	887	else
	888	{
	889	/* 32/64-bit ModR/M decode. */
	890	if ( modrm_rm == 4 )
	891	{
	892	sib = insn_fetch_type(uint8_t);
	893	sib_index = ((sib >> 3) & 7) \| ((rex_prefix << 2) & 8);
	894	sib_base = (sib & 7) \| ((rex_prefix << 3) & 8);
	895	if ( sib_index != 4 )
	896	ea.mem.off = (long)decode_register(sib_index, &_regs, 0);
	897	ea.mem.off <<= (sib >> 6) & 3;
	898	if ( (modrm_mod == 0) && ((sib_base & 7) == 5) )
	899	ea.mem.off += insn_fetch_type(int32_t);
	900	else if ( sib_base == 4 )
	901	{
	902	ea.mem.seg = x86_seg_ss;
	903	ea.mem.off += _regs.esp;
	904	if ( !twobyte && (b == 0x8f) )
	905	/* POP <rm> computes its EA post increment. */
	906	ea.mem.off += ((mode_64bit() && (op_bytes == 4))
	907	? 8 : op_bytes);
	908	}
	909	else if ( sib_base == 5 )
	910	{
	911	ea.mem.seg = x86_seg_ss;
	912	ea.mem.off += _regs.ebp;
	913	}
	914	else
	915	ea.mem.off += (long)decode_register(sib_base, &_regs, 0);
	916	}
	917	else
	918	{
	919	modrm_rm \|= (rex_prefix & 1) << 3;
	920	ea.mem.off = (long )decode_register(modrm_rm, &_regs, 0);
	921	if ( (modrm_rm == 5) && (modrm_mod != 0) )
	922	ea.mem.seg = x86_seg_ss;
	923	}
	924	switch ( modrm_mod )
	925	{
	926	case 0:
	927	if ( (modrm_rm & 7) != 5 )
	928	break;
	929	ea.mem.off = insn_fetch_type(int32_t);
	930	if ( !mode_64bit() )
	931	break;
	932	/* Relative to RIP of next instruction. Argh! */
	933	ea.mem.off += _regs.eip;
	934	if ( (d & SrcMask) == SrcImm )
	935	ea.mem.off += (d & ByteOp) ? 1 :
	936	((op_bytes == 8) ? 4 : op_bytes);
	937	else if ( (d & SrcMask) == SrcImmByte )
	938	ea.mem.off += 1;
	939	else if ( ((b == 0xf6) \|\| (b == 0xf7)) &&
	940	((modrm_reg & 7) <= 1) )
	941	/* Special case in Grp3: test has immediate operand. */
	942	ea.mem.off += (d & ByteOp) ? 1
	943	: ((op_bytes == 8) ? 4 : op_bytes);
	944	break;
	945	case 1:
	946	ea.mem.off += insn_fetch_type(int8_t);
	947	break;
	948	case 2:
	949	ea.mem.off += insn_fetch_type(int32_t);
	950	break;
	951	}
	952	ea.mem.off = truncate_ea(ea.mem.off);
	953	}
	954	}
	955
	956	if ( override_seg != -1 )
	957	ea.mem.seg = override_seg;
	958
	959	/* Special instructions do their own operand decoding. */
	960	if ( (d & DstMask) == ImplicitOps )
	961	goto special_insn;
	962
	963	/* Decode and fetch the source operand: register, memory or immediate. */
	964	switch ( d & SrcMask )
	965	{
	966	case SrcNone:
	967	break;
	968	case SrcReg:
	969	src.type = OP_REG;
	970	if ( d & ByteOp )
	971	{
	972	src.reg = decode_register(modrm_reg, &_regs, (rex_prefix == 0));
	973	src.val = (uint8_t )src.reg;
	974	src.bytes = 1;
	975	}
	976	else
	977	{
	978	src.reg = decode_register(modrm_reg, &_regs, 0);
	979	switch ( (src.bytes = op_bytes) )
	980	{
	981	case 2: src.val = (uint16_t )src.reg; break;
	982	case 4: src.val = (uint32_t )src.reg; break;
	983	case 8: src.val = (uint64_t )src.reg; break;
	984	}
	985	}
	986	break;
	987	case SrcMem16:
	988	ea.bytes = 2;
	989	goto srcmem_common;
	990	case SrcMem:
	991	ea.bytes = (d & ByteOp) ? 1 : op_bytes;
	992	srcmem_common:
	993	src = ea;
	994	if ( src.type == OP_REG )
	995	{
	996	switch ( src.bytes )
	997	{
	998	case 1: src.val = (uint8_t )src.reg; break;
	999	case 2: src.val = (uint16_t )src.reg; break;
	1000	case 4: src.val = (uint32_t )src.reg; break;
	1001	case 8: src.val = (uint64_t )src.reg; break;
	1002	}
	1003	}
	1004	else if ( (rc = ops->read(src.mem.seg, src.mem.off,
	1005	&src.val, src.bytes, ctxt)) )
	1006	goto done;
	1007	break;
	1008	case SrcImm:
	1009	src.type = OP_IMM;
	1010	src.bytes = (d & ByteOp) ? 1 : op_bytes;
	1011	if ( src.bytes == 8 ) src.bytes = 4;
	1012	/* NB. Immediates are sign-extended as necessary. */
	1013	switch ( src.bytes )
	1014	{
	1015	case 1: src.val = insn_fetch_type(int8_t); break;
	1016	case 2: src.val = insn_fetch_type(int16_t); break;
	1017	case 4: src.val = insn_fetch_type(int32_t); break;
	1018	}
	1019	break;
	1020	case SrcImmByte:
	1021	src.type = OP_IMM;
	1022	src.bytes = 1;
	1023	src.val = insn_fetch_type(int8_t);
	1024	break;
	1025	}
	1026
	1027	/* Decode and fetch the destination operand: register or memory. */
	1028	switch ( d & DstMask )
	1029	{
	1030	case DstReg:
	1031	dst.type = OP_REG;
	1032	if ( d & ByteOp )
	1033	{
	1034	dst.reg = decode_register(modrm_reg, &_regs, (rex_prefix == 0));
	1035	dst.val = (uint8_t )dst.reg;
	1036	dst.bytes = 1;
	1037	}
	1038	else
	1039	{
	1040	dst.reg = decode_register(modrm_reg, &_regs, 0);
	1041	switch ( (dst.bytes = op_bytes) )
	1042	{
	1043	case 2: dst.val = (uint16_t )dst.reg; break;
	1044	case 4: dst.val = (uint32_t )dst.reg; break;
	1045	case 8: dst.val = (uint64_t )dst.reg; break;
	1046	}
	1047	}
	1048	break;
	1049	case DstBitBase:
	1050	if ( ((d & SrcMask) == SrcImmByte) \|\| (ea.type == OP_REG) )
	1051	{
	1052	src.val &= (op_bytes << 3) - 1;
	1053	}
	1054	else
	1055	{
	1056	/*
	1057	* EA += BitOffset DIV op_bytes*8
	1058	* BitOffset = BitOffset MOD op_bytes*8
	1059	* DIV truncates towards negative infinity.
	1060	* MOD always produces a positive result.
	1061	*/
	1062	if ( op_bytes == 2 )
	1063	src.val = (int16_t)src.val;
	1064	else if ( op_bytes == 4 )
	1065	src.val = (int32_t)src.val;
	1066	if ( (long)src.val < 0 )
	1067	{
	1068	unsigned long byte_offset;
	1069	byte_offset = op_bytes + (((-src.val-1) >> 3) & ~(op_bytes-1));
	1070	ea.mem.off -= byte_offset;
	1071	src.val = (byte_offset << 3) + src.val;
	1072	}
	1073	else
	1074	{
	1075	ea.mem.off += (src.val >> 3) & ~(op_bytes - 1);
	1076	src.val &= (op_bytes << 3) - 1;
	1077	}
	1078	}
	1079	/* Becomes a normal DstMem operation from here on. */
	1080	d = (d & ~DstMask) \| DstMem;
	1081	case DstMem:
	1082	ea.bytes = (d & ByteOp) ? 1 : op_bytes;
	1083	dst = ea;
	1084	if ( dst.type == OP_REG )
	1085	{
	1086	switch ( dst.bytes )
	1087	{
	1088	case 1: dst.val = (uint8_t )dst.reg; break;
	1089	case 2: dst.val = (uint16_t )dst.reg; break;
	1090	case 4: dst.val = (uint32_t )dst.reg; break;
	1091	case 8: dst.val = (uint64_t )dst.reg; break;
	1092	}
	1093	}
	1094	else if ( !(d & Mov) ) /* optimisation - avoid slow emulated read */
	1095	{
	1096	if ( (rc = ops->read(dst.mem.seg, dst.mem.off,
	1097	&dst.val, dst.bytes, ctxt)) )
	1098	goto done;
	1099	dst.orig_val = dst.val;
	1100	}
	1101	break;
	1102	}
	1103
	1104	/* LOCK prefix allowed only on instructions with memory destination. */
	1105	generate_exception_if(lock_prefix && (dst.type != OP_MEM), EXC_GP);
	1106
	1107	if ( twobyte )
	1108	goto twobyte_insn;
	1109
	1110	switch ( b )
	1111	{
	1112	case 0x04 ... 0x05: /* add imm,%%eax */
	1113	dst.reg = (unsigned long *)&_regs.eax;
	1114	dst.val = _regs.eax;
	1115	case 0x00 ... 0x03: add: /* add */
	1116	emulate_2op_SrcV("add", src, dst, _regs.eflags);
	1117	break;
	1118
	1119	case 0x0c ... 0x0d: /* or imm,%%eax */
	1120	dst.reg = (unsigned long *)&_regs.eax;
	1121	dst.val = _regs.eax;
	1122	case 0x08 ... 0x0b: or: /* or */
	1123	emulate_2op_SrcV("or", src, dst, _regs.eflags);
	1124	break;
	1125
	1126	case 0x14 ... 0x15: /* adc imm,%%eax */
	1127	dst.reg = (unsigned long *)&_regs.eax;
	1128	dst.val = _regs.eax;
	1129	case 0x10 ... 0x13: adc: /* adc */
	1130	emulate_2op_SrcV("adc", src, dst, _regs.eflags);
	1131	break;
	1132
	1133	case 0x1c ... 0x1d: /* sbb imm,%%eax */
	1134	dst.reg = (unsigned long *)&_regs.eax;
	1135	dst.val = _regs.eax;
	1136	case 0x18 ... 0x1b: sbb: /* sbb */
	1137	emulate_2op_SrcV("sbb", src, dst, _regs.eflags);
	1138	break;
	1139
	1140	case 0x24 ... 0x25: /* and imm,%%eax */
	1141	dst.reg = (unsigned long *)&_regs.eax;
	1142	dst.val = _regs.eax;
	1143	case 0x20 ... 0x23: and: /* and */
	1144	emulate_2op_SrcV("and", src, dst, _regs.eflags);
	1145	break;
	1146
	1147	case 0x2c ... 0x2d: /* sub imm,%%eax */
	1148	dst.reg = (unsigned long *)&_regs.eax;
	1149	dst.val = _regs.eax;
	1150	case 0x28 ... 0x2b: sub: /* sub */
	1151	emulate_2op_SrcV("sub", src, dst, _regs.eflags);
	1152	break;
	1153
	1154	case 0x34 ... 0x35: /* xor imm,%%eax */
	1155	dst.reg = (unsigned long *)&_regs.eax;
	1156	dst.val = _regs.eax;
	1157	case 0x30 ... 0x33: xor: /* xor */
	1158	emulate_2op_SrcV("xor", src, dst, _regs.eflags);
	1159	break;
	1160
	1161	case 0x3c ... 0x3d: /* cmp imm,%%eax */
	1162	dst.reg = (unsigned long *)&_regs.eax;
	1163	dst.val = _regs.eax;
	1164	case 0x38 ... 0x3b: cmp: /* cmp */
	1165	emulate_2op_SrcV("cmp", src, dst, _regs.eflags);
	1166	break;
	1167
	1168	case 0x62: /* bound */ {
	1169	unsigned long src_val2;
	1170	int lb, ub, idx;
	1171	generate_exception_if(mode_64bit() \|\| (src.type != OP_MEM), EXC_UD);
	1172	if ( (rc = ops->read(src.mem.seg, src.mem.off + op_bytes,
	1173	&src_val2, op_bytes, ctxt)) )
	1174	goto done;
	1175	ub = (op_bytes == 2) ? (int16_t)src_val2 : (int32_t)src_val2;
	1176	lb = (op_bytes == 2) ? (int16_t)src.val : (int32_t)src.val;
	1177	idx = (op_bytes == 2) ? (int16_t)dst.val : (int32_t)dst.val;
	1178	generate_exception_if((idx < lb) \|\| (idx > ub), EXC_BR);
	1179	dst.type = OP_NONE;
	1180	break;
	1181	}
	1182
	1183	case 0x63: /* movsxd (x86/64) / arpl (x86/32) */
	1184	if ( mode_64bit() )
	1185	{
	1186	/* movsxd */
	1187	if ( src.type == OP_REG )
	1188	src.val = (int32_t )src.reg;
	1189	else if ( (rc = ops->read(src.mem.seg, src.mem.off,
	1190	&src.val, 4, ctxt)) )
	1191	goto done;
	1192	dst.val = (int32_t)src.val;
	1193	}
	1194	else
	1195	{
	1196	/* arpl */
	1197	uint16_t src_val = dst.val;
	1198	dst = src;
	1199	_regs.eflags &= ~EFLG_ZF;
	1200	_regs.eflags \|= ((src_val & 3) > (dst.val & 3)) ? EFLG_ZF : 0;
	1201	if ( _regs.eflags & EFLG_ZF )
	1202	dst.val = (dst.val & ~3) \| (src_val & 3);
	1203	else
	1204	dst.type = OP_NONE;
	1205	}
	1206	break;
	1207
	1208	case 0x69: /* imul imm16/32 */
	1209	case 0x6b: /* imul imm8 */ {
	1210	unsigned long reg = (long )decode_register(modrm_reg, &_regs, 0);
	1211	_regs.eflags &= ~(EFLG_OF\|EFLG_CF);
	1212	switch ( dst.bytes )
	1213	{
	1214	case 2:
	1215	dst.val = ((uint32_t)(int16_t)src.val *
	1216	(uint32_t)(int16_t)reg);
	1217	if ( (int16_t)dst.val != (uint32_t)dst.val )
	1218	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1219	break;
	1220	#ifdef __x86_64__
	1221	case 4:
	1222	dst.val = ((uint64_t)(int32_t)src.val *
	1223	(uint64_t)(int32_t)reg);
	1224	if ( (int32_t)dst.val != dst.val )
	1225	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1226	break;
	1227	#endif
	1228	default: {
	1229	unsigned long m[2] = { src.val, reg };
	1230	if ( imul_dbl(m) )
	1231	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1232	dst.val = m[0];
	1233	break;
	1234	}
	1235	}
	1236	dst.type = OP_REG;
	1237	dst.reg = decode_register(modrm_reg, &_regs, 0);
	1238	break;
	1239	}
	1240
	1241	case 0x82: /* Grp1 (x86/32 only) */
	1242	generate_exception_if(mode_64bit(), EXC_UD);
	1243	case 0x80: case 0x81: case 0x83: /* Grp1 */
	1244	switch ( modrm_reg & 7 )
	1245	{
	1246	case 0: goto add;
	1247	case 1: goto or;
	1248	case 2: goto adc;
	1249	case 3: goto sbb;
	1250	case 4: goto and;
	1251	case 5: goto sub;
	1252	case 6: goto xor;
	1253	case 7: goto cmp;
	1254	}
	1255	break;
	1256
	1257	case 0xa8 ... 0xa9: /* test imm,%%eax */
	1258	dst.reg = (unsigned long *)&_regs.eax;
	1259	dst.val = _regs.eax;
	1260	case 0x84 ... 0x85: test: /* test */
	1261	emulate_2op_SrcV("test", src, dst, _regs.eflags);
	1262	break;
	1263
	1264	case 0x86 ... 0x87: xchg: /* xchg */
	1265	/* Write back the register source. */
	1266	switch ( dst.bytes )
	1267	{
	1268	case 1: (uint8_t )src.reg = (uint8_t)dst.val; break;
	1269	case 2: (uint16_t )src.reg = (uint16_t)dst.val; break;
	1270	case 4: src.reg = (uint32_t)dst.val; break; / 64b reg: zero-extend */
	1271	case 8: *src.reg = dst.val; break;
	1272	}
	1273	/* Write back the memory destination with implicit LOCK prefix. */
	1274	dst.val = src.val;
	1275	lock_prefix = 1;
	1276	break;
	1277
	1278	case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
	1279	generate_exception_if((modrm_reg & 7) != 0, EXC_UD);
	1280	case 0x88 ... 0x8b: /* mov */
	1281	dst.val = src.val;
	1282	break;
	1283
	1284	case 0x8d: /* lea */
	1285	dst.val = ea.mem.off;
	1286	break;
	1287
	1288	case 0x8f: /* pop (sole member of Grp1a) */
	1289	generate_exception_if((modrm_reg & 7) != 0, EXC_UD);
	1290	/* 64-bit mode: POP defaults to a 64-bit operand. */
	1291	if ( mode_64bit() && (dst.bytes == 4) )
	1292	dst.bytes = 8;
	1293	if ( (rc = ops->read(x86_seg_ss, sp_post_inc(dst.bytes),
	1294	&dst.val, dst.bytes, ctxt)) != 0 )
	1295	goto done;
	1296	break;
	1297
	1298	case 0xb0 ... 0xb7: /* mov imm8,r8 */
	1299	dst.reg = decode_register(
	1300	(b & 7) \| ((rex_prefix & 1) << 3), &_regs, (rex_prefix == 0));
	1301	dst.val = src.val;
	1302	break;
	1303
	1304	case 0xb8 ... 0xbf: /* mov imm{16,32,64},r{16,32,64} */
	1305	if ( dst.bytes == 8 ) /* Fetch more bytes to obtain imm64 */
	1306	src.val = ((uint32_t)src.val \|
	1307	((uint64_t)insn_fetch_type(uint32_t) << 32));
	1308	dst.reg = decode_register(
	1309	(b & 7) \| ((rex_prefix & 1) << 3), &_regs, 0);
	1310	dst.val = src.val;
	1311	break;
	1312
	1313	case 0xc0 ... 0xc1: grp2: /* Grp2 */
	1314	switch ( modrm_reg & 7 )
	1315	{
	1316	case 0: /* rol */
	1317	emulate_2op_SrcB("rol", src, dst, _regs.eflags);
	1318	break;
	1319	case 1: /* ror */
	1320	emulate_2op_SrcB("ror", src, dst, _regs.eflags);
	1321	break;
	1322	case 2: /* rcl */
	1323	emulate_2op_SrcB("rcl", src, dst, _regs.eflags);
	1324	break;
	1325	case 3: /* rcr */
	1326	emulate_2op_SrcB("rcr", src, dst, _regs.eflags);
	1327	break;
	1328	case 4: /* sal/shl */
	1329	case 6: /* sal/shl */
	1330	emulate_2op_SrcB("sal", src, dst, _regs.eflags);
	1331	break;
	1332	case 5: /* shr */
	1333	emulate_2op_SrcB("shr", src, dst, _regs.eflags);
	1334	break;
	1335	case 7: /* sar */
	1336	emulate_2op_SrcB("sar", src, dst, _regs.eflags);
	1337	break;
	1338	}
	1339	break;
	1340
	1341	case 0xd0 ... 0xd1: /* Grp2 */
	1342	src.val = 1;
	1343	goto grp2;
	1344
	1345	case 0xd2 ... 0xd3: /* Grp2 */
	1346	src.val = _regs.ecx;
	1347	goto grp2;
	1348
	1349	case 0xf6 ... 0xf7: /* Grp3 */
	1350	switch ( modrm_reg & 7 )
	1351	{
	1352	case 0 ... 1: /* test */
	1353	/* Special case in Grp3: test has an immediate source operand. */
	1354	src.type = OP_IMM;
	1355	src.bytes = (d & ByteOp) ? 1 : op_bytes;
	1356	if ( src.bytes == 8 ) src.bytes = 4;
	1357	switch ( src.bytes )
	1358	{
	1359	case 1: src.val = insn_fetch_type(int8_t); break;
	1360	case 2: src.val = insn_fetch_type(int16_t); break;
	1361	case 4: src.val = insn_fetch_type(int32_t); break;
	1362	}
	1363	goto test;
	1364	case 2: /* not */
	1365	dst.val = ~dst.val;
	1366	break;
	1367	case 3: /* neg */
	1368	emulate_1op("neg", dst, _regs.eflags);
	1369	break;
	1370	case 4: /* mul */
	1371	src = dst;
	1372	dst.type = OP_REG;
	1373	dst.reg = (unsigned long *)&_regs.eax;
	1374	dst.val = *dst.reg;
	1375	_regs.eflags &= ~(EFLG_OF\|EFLG_CF);
	1376	switch ( src.bytes )
	1377	{
	1378	case 1:
	1379	dst.val *= src.val;
	1380	if ( (uint8_t)dst.val != (uint16_t)dst.val )
	1381	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1382	break;
	1383	case 2:
	1384	dst.val *= src.val;
	1385	if ( (uint16_t)dst.val != (uint32_t)dst.val )
	1386	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1387	(uint16_t )&_regs.edx = dst.val >> 16;
	1388	break;
	1389	#ifdef __x86_64__
	1390	case 4:
	1391	dst.val *= src.val;
	1392	if ( (uint32_t)dst.val != dst.val )
	1393	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1394	_regs.edx = (uint32_t)(dst.val >> 32);
	1395	break;
	1396	#endif
	1397	default: {
	1398	unsigned long m[2] = { src.val, dst.val };
	1399	if ( mul_dbl(m) )
	1400	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1401	_regs.edx = m[1];
	1402	dst.val = m[0];
	1403	break;
	1404	}
	1405	}
	1406	break;
	1407	case 5: /* imul */
	1408	src = dst;
	1409	dst.type = OP_REG;
	1410	dst.reg = (unsigned long *)&_regs.eax;
	1411	dst.val = *dst.reg;
	1412	_regs.eflags &= ~(EFLG_OF\|EFLG_CF);
	1413	switch ( src.bytes )
	1414	{
	1415	case 1:
	1416	dst.val = ((uint16_t)(int8_t)src.val *
	1417	(uint16_t)(int8_t)dst.val);
	1418	if ( (int8_t)dst.val != (uint16_t)dst.val )
	1419	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1420	break;
	1421	case 2:
	1422	dst.val = ((uint32_t)(int16_t)src.val *
	1423	(uint32_t)(int16_t)dst.val);
	1424	if ( (int16_t)dst.val != (uint32_t)dst.val )
	1425	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1426	(uint16_t )&_regs.edx = dst.val >> 16;
	1427	break;
	1428	#ifdef __x86_64__
	1429	case 4:
	1430	dst.val = ((uint64_t)(int32_t)src.val *
	1431	(uint64_t)(int32_t)dst.val);
	1432	if ( (int32_t)dst.val != dst.val )
	1433	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1434	_regs.edx = (uint32_t)(dst.val >> 32);
	1435	break;
	1436	#endif
	1437	default: {
	1438	unsigned long m[2] = { src.val, dst.val };
	1439	if ( imul_dbl(m) )
	1440	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	1441	_regs.edx = m[1];
	1442	dst.val = m[0];
	1443	break;
	1444	}
	1445	}
	1446	break;
	1447	case 6: /* div */ {
	1448	unsigned long u[2], v;
	1449	src = dst;
	1450	dst.type = OP_REG;
	1451	dst.reg = (unsigned long *)&_regs.eax;
	1452	switch ( src.bytes )
	1453	{
	1454	case 1:
	1455	u[0] = (uint16_t)_regs.eax;
	1456	u[1] = 0;
	1457	v = (uint8_t)src.val;
	1458	generate_exception_if(
	1459	div_dbl(u, v) \|\| ((uint8_t)u[0] != (uint16_t)u[0]),
	1460	EXC_DE);
	1461	dst.val = (uint8_t)u[0];
	1462	((uint8_t *)&_regs.eax)[1] = u[1];
	1463	break;
	1464	case 2:
	1465	u[0] = ((uint32_t)_regs.edx << 16) \| (uint16_t)_regs.eax;
	1466	u[1] = 0;
	1467	v = (uint16_t)src.val;
	1468	generate_exception_if(
	1469	div_dbl(u, v) \|\| ((uint16_t)u[0] != (uint32_t)u[0]),
	1470	EXC_DE);
	1471	dst.val = (uint16_t)u[0];
	1472	(uint16_t )&_regs.edx = u[1];
	1473	break;
	1474	#ifdef __x86_64__
	1475	case 4:
	1476	u[0] = (_regs.edx << 32) \| (uint32_t)_regs.eax;
	1477	u[1] = 0;
	1478	v = (uint32_t)src.val;
	1479	generate_exception_if(
	1480	div_dbl(u, v) \|\| ((uint32_t)u[0] != u[0]),
	1481	EXC_DE);
	1482	dst.val = (uint32_t)u[0];
	1483	_regs.edx = (uint32_t)u[1];
	1484	break;
	1485	#endif
	1486	default:
	1487	u[0] = _regs.eax;
	1488	u[1] = _regs.edx;
	1489	v = src.val;
	1490	generate_exception_if(div_dbl(u, v), EXC_DE);
	1491	dst.val = u[0];
	1492	_regs.edx = u[1];
	1493	break;
	1494	}
	1495	break;
	1496	}
	1497	case 7: /* idiv */ {
	1498	unsigned long u[2], v;
	1499	src = dst;
	1500	dst.type = OP_REG;
	1501	dst.reg = (unsigned long *)&_regs.eax;
	1502	switch ( src.bytes )
	1503	{
	1504	case 1:
	1505	u[0] = (int16_t)_regs.eax;
	1506	u[1] = ((long)u[0] < 0) ? ~0UL : 0UL;
	1507	v = (int8_t)src.val;
	1508	generate_exception_if(
	1509	idiv_dbl(u, v) \|\| ((int8_t)u[0] != (int16_t)u[0]),
	1510	EXC_DE);
	1511	dst.val = (int8_t)u[0];
	1512	((int8_t *)&_regs.eax)[1] = u[1];
	1513	break;
	1514	case 2:
	1515	u[0] = (int32_t)((_regs.edx << 16) \| (uint16_t)_regs.eax);
	1516	u[1] = ((long)u[0] < 0) ? ~0UL : 0UL;
	1517	v = (int16_t)src.val;
	1518	generate_exception_if(
	1519	idiv_dbl(u, v) \|\| ((int16_t)u[0] != (int32_t)u[0]),
	1520	EXC_DE);
	1521	dst.val = (int16_t)u[0];
	1522	(int16_t )&_regs.edx = u[1];
	1523	break;
	1524	#ifdef __x86_64__
	1525	case 4:
	1526	u[0] = (_regs.edx << 32) \| (uint32_t)_regs.eax;
	1527	u[1] = ((long)u[0] < 0) ? ~0UL : 0UL;
	1528	v = (int32_t)src.val;
	1529	generate_exception_if(
	1530	idiv_dbl(u, v) \|\| ((int32_t)u[0] != u[0]),
	1531	EXC_DE);
	1532	dst.val = (int32_t)u[0];
	1533	_regs.edx = (uint32_t)u[1];
	1534	break;
	1535	#endif
	1536	default:
	1537	u[0] = _regs.eax;
	1538	u[1] = _regs.edx;
	1539	v = src.val;
	1540	generate_exception_if(idiv_dbl(u, v), EXC_DE);
	1541	dst.val = u[0];
	1542	_regs.edx = u[1];
	1543	break;
	1544	}
	1545	break;
	1546	}
	1547	default:
	1548	goto cannot_emulate;
	1549	}
	1550	break;
	1551
	1552	case 0xfe: /* Grp4 */
	1553	generate_exception_if((modrm_reg & 7) >= 2, EXC_UD);
	1554	case 0xff: /* Grp5 */
	1555	switch ( modrm_reg & 7 )
	1556	{
	1557	case 0: /* inc */
	1558	emulate_1op("inc", dst, _regs.eflags);
	1559	break;
	1560	case 1: /* dec */
	1561	emulate_1op("dec", dst, _regs.eflags);
	1562	break;
	1563	case 2: /* call (near) */
	1564	case 4: /* jmp (near) */
	1565	if ( ((op_bytes = dst.bytes) != 8) && mode_64bit() )
	1566	{
	1567	dst.bytes = op_bytes = 8;
	1568	if ( dst.type == OP_REG )
	1569	dst.val = *dst.reg;
	1570	else if ( (rc = ops->read(dst.mem.seg, dst.mem.off,
	1571	&dst.val, 8, ctxt)) != 0 )
	1572	goto done;
	1573	}
	1574	src.val = _regs.eip;
	1575	_regs.eip = dst.val;
	1576	if ( (modrm_reg & 7) == 2 )
	1577	goto push; /* call */
	1578	break;
	1579	case 6: /* push */
	1580	/* 64-bit mode: PUSH defaults to a 64-bit operand. */
	1581	if ( mode_64bit() && (dst.bytes == 4) )
	1582	{
	1583	dst.bytes = 8;
	1584	if ( dst.type == OP_REG )
	1585	dst.val = *dst.reg;
	1586	else if ( (rc = ops->read(dst.mem.seg, dst.mem.off,
	1587	&dst.val, 8, ctxt)) != 0 )
	1588	goto done;
	1589	}
	1590	if ( (rc = ops->write(x86_seg_ss, sp_pre_dec(dst.bytes),
	1591	dst.val, dst.bytes, ctxt)) != 0 )
	1592	goto done;
	1593	dst.type = OP_NONE;
	1594	break;
	1595	case 7:
	1596	generate_exception_if(1, EXC_UD);
	1597	default:
	1598	goto cannot_emulate;
	1599	}
	1600	break;
	1601	}
	1602
	1603	writeback:
	1604	switch ( dst.type )
	1605	{
	1606	case OP_REG:
	1607	/* The 4-byte case is correct: in 64-bit mode we zero-extend. */
	1608	switch ( dst.bytes )
	1609	{
	1610	case 1: (uint8_t )dst.reg = (uint8_t)dst.val; break;
	1611	case 2: (uint16_t )dst.reg = (uint16_t)dst.val; break;
	1612	case 4: dst.reg = (uint32_t)dst.val; break; / 64b: zero-ext */
	1613	case 8: *dst.reg = dst.val; break;
	1614	}
	1615	break;
	1616	case OP_MEM:
	1617	if ( !(d & Mov) && (dst.orig_val == dst.val) )
	1618	/* nothing to do */;
	1619	else if ( lock_prefix )
	1620	rc = ops->cmpxchg(
	1621	dst.mem.seg, dst.mem.off, dst.orig_val,
	1622	dst.val, dst.bytes, ctxt);
	1623	else
	1624	rc = ops->write(
	1625	dst.mem.seg, dst.mem.off, dst.val, dst.bytes, ctxt);
	1626	if ( rc != 0 )
	1627	goto done;
	1628	default:
	1629	break;
	1630	}
	1631
	1632	/* Commit shadow register state. */
	1633	*ctxt->regs = _regs;
	1634
	1635	done:
	1636	return rc;
	1637
	1638	special_insn:
	1639	dst.type = OP_NONE;
	1640
	1641	/*
	1642	* The only implicit-operands instructions allowed a LOCK prefix are
	1643	* CMPXCHG{8,16}B, MOV CRn, MOV DRn.
	1644	*/
	1645	generate_exception_if(lock_prefix &&
	1646	((b < 0x20) \|\| (b > 0x23)) && /* MOV CRn/DRn */
	1647	(b != 0xc7), /* CMPXCHG{8,16}B */
	1648	EXC_GP);
	1649
	1650	if ( twobyte )
	1651	goto twobyte_special_insn;
	1652
	1653	switch ( b )
	1654	{
	1655	case 0x27: /* daa */ {
	1656	uint8_t al = _regs.eax;
	1657	unsigned long eflags = _regs.eflags;
	1658	generate_exception_if(mode_64bit(), EXC_UD);
	1659	_regs.eflags &= ~(EFLG_CF\|EFLG_AF);
	1660	if ( ((al & 0x0f) > 9) \|\| (eflags & EFLG_AF) )
	1661	{
	1662	(uint8_t )&_regs.eax += 6;
	1663	_regs.eflags \|= EFLG_AF;
	1664	}
	1665	if ( (al > 0x99) \|\| (eflags & EFLG_CF) )
	1666	{
	1667	(uint8_t )&_regs.eax += 0x60;
	1668	_regs.eflags \|= EFLG_CF;
	1669	}
	1670	_regs.eflags &= ~(EFLG_SF\|EFLG_ZF\|EFLG_PF);
	1671	_regs.eflags \|= ((uint8_t)_regs.eax == 0) ? EFLG_ZF : 0;
	1672	_regs.eflags \|= (( int8_t)_regs.eax < 0) ? EFLG_SF : 0;
	1673	_regs.eflags \|= even_parity(_regs.eax) ? EFLG_PF : 0;
	1674	break;
	1675	}
	1676
	1677	case 0x2f: /* das */ {
	1678	uint8_t al = _regs.eax;
	1679	unsigned long eflags = _regs.eflags;
	1680	generate_exception_if(mode_64bit(), EXC_UD);
	1681	_regs.eflags &= ~(EFLG_CF\|EFLG_AF);
	1682	if ( ((al & 0x0f) > 9) \|\| (eflags & EFLG_AF) )
	1683	{
	1684	_regs.eflags \|= EFLG_AF;
	1685	if ( (al < 6) \|\| (eflags & EFLG_CF) )
	1686	_regs.eflags \|= EFLG_CF;
	1687	(uint8_t )&_regs.eax -= 6;
	1688	}
	1689	if ( (al > 0x99) \|\| (eflags & EFLG_CF) )
	1690	{
	1691	(uint8_t )&_regs.eax -= 0x60;
	1692	_regs.eflags \|= EFLG_CF;
	1693	}
	1694	_regs.eflags &= ~(EFLG_SF\|EFLG_ZF\|EFLG_PF);
	1695	_regs.eflags \|= ((uint8_t)_regs.eax == 0) ? EFLG_ZF : 0;
	1696	_regs.eflags \|= (( int8_t)_regs.eax < 0) ? EFLG_SF : 0;
	1697	_regs.eflags \|= even_parity(_regs.eax) ? EFLG_PF : 0;
	1698	break;
	1699	}
	1700
	1701	case 0x37: /* aaa */
	1702	case 0x3f: /* aas */
	1703	generate_exception_if(mode_64bit(), EXC_UD);
	1704	_regs.eflags &= ~EFLG_CF;
	1705	if ( ((uint8_t)_regs.eax > 9) \|\| (_regs.eflags & EFLG_AF) )
	1706	{
	1707	((uint8_t *)&_regs.eax)[0] += (b == 0x37) ? 6 : -6;
	1708	((uint8_t *)&_regs.eax)[1] += (b == 0x37) ? 1 : -1;
	1709	_regs.eflags \|= EFLG_CF \| EFLG_AF;
	1710	}
	1711	((uint8_t *)&_regs.eax)[0] &= 0x0f;
	1712	break;
	1713
	1714	case 0x40 ... 0x4f: /* inc/dec reg */
	1715	dst.type = OP_REG;
	1716	dst.reg = decode_register(b & 7, &_regs, 0);
	1717	dst.bytes = op_bytes;
	1718	dst.val = *dst.reg;
	1719	if ( b & 8 )
	1720	emulate_1op("dec", dst, _regs.eflags);
	1721	else
	1722	emulate_1op("inc", dst, _regs.eflags);
	1723	break;
	1724
	1725	case 0x50 ... 0x57: /* push reg */
	1726	src.val = (unsigned long )decode_register(
	1727	(b & 7) \| ((rex_prefix & 1) << 3), &_regs, 0);
	1728	goto push;
	1729
	1730	case 0x58 ... 0x5f: /* pop reg */
	1731	dst.type = OP_REG;
	1732	dst.reg = decode_register(
	1733	(b & 7) \| ((rex_prefix & 1) << 3), &_regs, 0);
	1734	dst.bytes = op_bytes;
	1735	if ( mode_64bit() && (dst.bytes == 4) )
	1736	dst.bytes = 8;
	1737	if ( (rc = ops->read(x86_seg_ss, sp_post_inc(dst.bytes),
	1738	&dst.val, dst.bytes, ctxt)) != 0 )
	1739	goto done;
	1740	break;
	1741
	1742	case 0x60: /* pusha */ {
	1743	int i;
	1744	unsigned long regs[] = {
	1745	_regs.eax, _regs.ecx, _regs.edx, _regs.ebx,
	1746	_regs.esp, _regs.ebp, _regs.esi, _regs.edi };
	1747	generate_exception_if(mode_64bit(), EXC_UD);
	1748	for ( i = 0; i < 8; i++ )
	1749	if ( (rc = ops->write(x86_seg_ss, sp_pre_dec(op_bytes),
	1750	regs[i], op_bytes, ctxt)) != 0 )
	1751	goto done;
	1752	break;
	1753	}
	1754
	1755	case 0x61: /* popa */ {
	1756	int i;
	1757	unsigned long dummy_esp, *regs[] = {
	1758	(unsigned long )&_regs.edi, (unsigned long )&_regs.esi,
	1759	(unsigned long )&_regs.ebp, (unsigned long )&dummy_esp,
	1760	(unsigned long )&_regs.ebx, (unsigned long )&_regs.edx,
	1761	(unsigned long )&_regs.ecx, (unsigned long )&_regs.eax };
	1762	generate_exception_if(mode_64bit(), EXC_UD);
	1763	for ( i = 0; i < 8; i++ )
	1764	if ( (rc = ops->read(x86_seg_ss, sp_post_inc(op_bytes),
	1765	regs[i], op_bytes, ctxt)) != 0 )
	1766	goto done;
	1767	break;
	1768	}
	1769
	1770	case 0x68: /* push imm{16,32,64} */
	1771	src.val = ((op_bytes == 2)
	1772	? (int32_t)insn_fetch_type(int16_t)
	1773	: insn_fetch_type(int32_t));
	1774	goto push;
	1775
	1776	case 0x6a: /* push imm8 */
	1777	src.val = insn_fetch_type(int8_t);
	1778	push:
	1779	d \|= Mov; /* force writeback */
	1780	dst.type = OP_MEM;
	1781	dst.bytes = op_bytes;
	1782	if ( mode_64bit() && (dst.bytes == 4) )
	1783	dst.bytes = 8;
	1784	dst.val = src.val;
	1785	dst.mem.seg = x86_seg_ss;
	1786	dst.mem.off = sp_pre_dec(dst.bytes);
	1787	break;
	1788
	1789	case 0x6c ... 0x6d: /* ins %dx,%es:%edi */
	1790	handle_rep_prefix();
	1791	generate_exception_if(!mode_iopl(), EXC_GP);
	1792	dst.type = OP_MEM;
	1793	dst.bytes = !(b & 1) ? 1 : (op_bytes == 8) ? 4 : op_bytes;
	1794	dst.mem.seg = x86_seg_es;
	1795	dst.mem.off = truncate_ea(_regs.edi);
	1796	fail_if(ops->read_io == NULL);
	1797	if ( (rc = ops->read_io((uint16_t)_regs.edx, dst.bytes,
	1798	&dst.val, ctxt)) != 0 )
	1799	goto done;
	1800	register_address_increment(
	1801	_regs.edi, (_regs.eflags & EFLG_DF) ? -dst.bytes : dst.bytes);
	1802	break;
	1803
	1804	case 0x6e ... 0x6f: /* outs %esi,%dx */
	1805	handle_rep_prefix();
	1806	generate_exception_if(!mode_iopl(), EXC_GP);
	1807	dst.bytes = !(b & 1) ? 1 : (op_bytes == 8) ? 4 : op_bytes;
	1808	if ( (rc = ops->read(ea.mem.seg, truncate_ea(_regs.esi),
	1809	&dst.val, dst.bytes, ctxt)) != 0 )
	1810	goto done;
	1811	fail_if(ops->write_io == NULL);
	1812	if ( (rc = ops->write_io((uint16_t)_regs.edx, dst.bytes,
	1813	dst.val, ctxt)) != 0 )
	1814	goto done;
	1815	register_address_increment(
	1816	_regs.esi, (_regs.eflags & EFLG_DF) ? -dst.bytes : dst.bytes);
	1817	break;
	1818
	1819	case 0x70 ... 0x7f: /* jcc (short) */ {
	1820	int rel = insn_fetch_type(int8_t);
	1821	if ( test_cc(b, _regs.eflags) )
	1822	jmp_rel(rel);
	1823	break;
	1824	}
	1825
	1826	case 0x90: /* nop / xchg %%r8,%%rax */
	1827	if ( !(rex_prefix & 1) )
	1828	break; /* nop */
	1829
	1830	case 0x91 ... 0x97: /* xchg reg,%%rax */
	1831	src.type = dst.type = OP_REG;
	1832	src.bytes = dst.bytes = op_bytes;
	1833	src.reg = (unsigned long *)&_regs.eax;
	1834	src.val = *src.reg;
	1835	dst.reg = decode_register(
	1836	(b & 7) \| ((rex_prefix & 1) << 3), &_regs, 0);
	1837	dst.val = *dst.reg;
	1838	goto xchg;
	1839
	1840	case 0x98: /* cbw/cwde/cdqe */
	1841	switch ( op_bytes )
	1842	{
	1843	case 2: (int16_t )&_regs.eax = (int8_t)_regs.eax; break; /* cbw */
	1844	case 4: _regs.eax = (uint32_t)(int16_t)_regs.eax; break; /* cwde */
	1845	case 8: _regs.eax = (int32_t)_regs.eax; break; /* cdqe */
	1846	}
	1847	break;
	1848
	1849	case 0x99: /* cwd/cdq/cqo */
	1850	switch ( op_bytes )
	1851	{
	1852	case 2:
	1853	(int16_t )&_regs.edx = ((int16_t)_regs.eax < 0) ? -1 : 0;
	1854	break;
	1855	case 4:
	1856	_regs.edx = (uint32_t)(((int32_t)_regs.eax < 0) ? -1 : 0);
	1857	break;
	1858	case 8:
	1859	_regs.edx = (_regs.eax < 0) ? -1 : 0;
	1860	break;
	1861	}
	1862	break;
	1863
	1864	case 0x9e: /* sahf */
	1865	(uint8_t )_regs.eflags = (((uint8_t *)&_regs.eax)[1] & 0xd7) \| 0x02;
	1866	break;
	1867
	1868	case 0x9f: /* lahf */
	1869	((uint8_t *)&_regs.eax)[1] = (_regs.eflags & 0xd7) \| 0x02;
	1870	break;
	1871
	1872	case 0xa0 ... 0xa1: /* mov mem.offs,{%al,%ax,%eax,%rax} */
	1873	/* Source EA is not encoded via ModRM. */
	1874	dst.type = OP_REG;
	1875	dst.reg = (unsigned long *)&_regs.eax;
	1876	dst.bytes = (d & ByteOp) ? 1 : op_bytes;
	1877	if ( (rc = ops->read(ea.mem.seg, insn_fetch_bytes(ad_bytes),
	1878	&dst.val, dst.bytes, ctxt)) != 0 )
	1879	goto done;
	1880	break;
	1881
	1882	case 0xa2 ... 0xa3: /* mov {%al,%ax,%eax,%rax},mem.offs */
	1883	/* Destination EA is not encoded via ModRM. */
	1884	dst.type = OP_MEM;
	1885	dst.mem.seg = ea.mem.seg;
	1886	dst.mem.off = insn_fetch_bytes(ad_bytes);
	1887	dst.bytes = (d & ByteOp) ? 1 : op_bytes;
	1888	dst.val = (unsigned long)_regs.eax;
	1889	break;
	1890
	1891	case 0xa4 ... 0xa5: /* movs */
	1892	handle_rep_prefix();
	1893	dst.type = OP_MEM;
	1894	dst.bytes = (d & ByteOp) ? 1 : op_bytes;
	1895	dst.mem.seg = x86_seg_es;
	1896	dst.mem.off = truncate_ea(_regs.edi);
	1897	if ( (rc = ops->read(ea.mem.seg, truncate_ea(_regs.esi),
	1898	&dst.val, dst.bytes, ctxt)) != 0 )
	1899	goto done;
	1900	register_address_increment(
	1901	_regs.esi, (_regs.eflags & EFLG_DF) ? -dst.bytes : dst.bytes);
	1902	register_address_increment(
	1903	_regs.edi, (_regs.eflags & EFLG_DF) ? -dst.bytes : dst.bytes);
	1904	break;
	1905
	1906	case 0xaa ... 0xab: /* stos */
	1907	handle_rep_prefix();
	1908	dst.type = OP_MEM;
	1909	dst.bytes = (d & ByteOp) ? 1 : op_bytes;
	1910	dst.mem.seg = x86_seg_es;
	1911	dst.mem.off = truncate_ea(_regs.edi);
	1912	dst.val = _regs.eax;
	1913	register_address_increment(
	1914	_regs.edi, (_regs.eflags & EFLG_DF) ? -dst.bytes : dst.bytes);
	1915	break;
	1916
	1917	case 0xac ... 0xad: /* lods */
	1918	handle_rep_prefix();
	1919	dst.type = OP_REG;
	1920	dst.bytes = (d & ByteOp) ? 1 : op_bytes;
	1921	dst.reg = (unsigned long *)&_regs.eax;
	1922	if ( (rc = ops->read(ea.mem.seg, truncate_ea(_regs.esi),
	1923	&dst.val, dst.bytes, ctxt)) != 0 )
	1924	goto done;
	1925	register_address_increment(
	1926	_regs.esi, (_regs.eflags & EFLG_DF) ? -dst.bytes : dst.bytes);
	1927	break;
	1928
	1929	case 0xc2: /* ret imm16 (near) */
	1930	case 0xc3: /* ret (near) */ {
	1931	int offset = (b == 0xc2) ? insn_fetch_type(uint16_t) : 0;
	1932	op_bytes = mode_64bit() ? 8 : op_bytes;
	1933	if ( (rc = ops->read(x86_seg_ss, sp_post_inc(op_bytes + offset),
	1934	&dst.val, op_bytes, ctxt)) != 0 )
	1935	goto done;
	1936	_regs.eip = dst.val;
	1937	break;
	1938	}
	1939
	1940	case 0xd4: /* aam */ {
	1941	unsigned int base = insn_fetch_type(uint8_t);
	1942	uint8_t al = _regs.eax;
	1943	generate_exception_if(mode_64bit(), EXC_UD);
	1944	generate_exception_if(base == 0, EXC_DE);
	1945	(uint16_t )&_regs.eax = ((al / base) << 8) \| (al % base);
	1946	_regs.eflags &= ~(EFLG_SF\|EFLG_ZF\|EFLG_PF);
	1947	_regs.eflags \|= ((uint8_t)_regs.eax == 0) ? EFLG_ZF : 0;
	1948	_regs.eflags \|= (( int8_t)_regs.eax < 0) ? EFLG_SF : 0;
	1949	_regs.eflags \|= even_parity(_regs.eax) ? EFLG_PF : 0;
	1950	break;
	1951	}
	1952
	1953	case 0xd5: /* aad */ {
	1954	unsigned int base = insn_fetch_type(uint8_t);
	1955	uint16_t ax = _regs.eax;
	1956	generate_exception_if(mode_64bit(), EXC_UD);
	1957	(uint16_t )&_regs.eax = (uint8_t)(ax + ((ax >> 8) * base));
	1958	_regs.eflags &= ~(EFLG_SF\|EFLG_ZF\|EFLG_PF);
	1959	_regs.eflags \|= ((uint8_t)_regs.eax == 0) ? EFLG_ZF : 0;
	1960	_regs.eflags \|= (( int8_t)_regs.eax < 0) ? EFLG_SF : 0;
	1961	_regs.eflags \|= even_parity(_regs.eax) ? EFLG_PF : 0;
	1962	break;
	1963	}
	1964
	1965	case 0xd6: /* salc */
	1966	generate_exception_if(mode_64bit(), EXC_UD);
	1967	(uint8_t )&_regs.eax = (_regs.eflags & EFLG_CF) ? 0xff : 0x00;
	1968	break;
	1969
	1970	case 0xd7: /* xlat */ {
	1971	unsigned long al = (uint8_t)_regs.eax;
	1972	if ( (rc = ops->read(ea.mem.seg, truncate_ea(_regs.ebx + al),
	1973	&al, 1, ctxt)) != 0 )
	1974	goto done;
	1975	(uint8_t )&_regs.eax = al;
	1976	break;
	1977	}
	1978
	1979	case 0xe0 ... 0xe2: /* loop{,z,nz} */ {
	1980	int rel = insn_fetch_type(int8_t);
	1981	int do_jmp = !(_regs.eflags & EFLG_ZF); /* loopnz */
	1982	if ( b == 0xe1 )
	1983	do_jmp = !do_jmp; /* loopz */
	1984	else if ( b == 0xe2 )
	1985	do_jmp = 1; /* loop */
	1986	switch ( ad_bytes )
	1987	{
	1988	case 2:
	1989	do_jmp &= --((uint16_t )&_regs.ecx) != 0;
	1990	break;
	1991	case 4:
	1992	do_jmp &= --((uint32_t )&_regs.ecx) != 0;
	1993	_regs.ecx = (uint32_t)_regs.ecx; /* zero extend in x86/64 mode */
	1994	break;
	1995	default: /* case 8: */
	1996	do_jmp &= --_regs.ecx != 0;
	1997	break;
	1998	}
	1999	if ( do_jmp )
	2000	jmp_rel(rel);
	2001	break;
	2002	}
	2003
	2004	case 0xe3: /* jcxz/jecxz (short) */ {
	2005	int rel = insn_fetch_type(int8_t);
	2006	if ( (ad_bytes == 2) ? !(uint16_t)_regs.ecx :
	2007	(ad_bytes == 4) ? !(uint32_t)_regs.ecx : !_regs.ecx )
	2008	jmp_rel(rel);
	2009	break;
	2010	}
	2011
	2012	case 0xe4: /* in imm8,%al */
	2013	case 0xe5: /* in imm8,%eax */
	2014	case 0xe6: /* out %al,imm8 */
	2015	case 0xe7: /* out %eax,imm8 */
	2016	case 0xec: /* in %dx,%al */
	2017	case 0xed: /* in %dx,%eax */
	2018	case 0xee: /* out %al,%dx */
	2019	case 0xef: /* out %eax,%dx */ {
	2020	unsigned int port = ((b < 0xe8)
	2021	? insn_fetch_type(uint8_t)
	2022	: (uint16_t)_regs.edx);
	2023	generate_exception_if(!mode_iopl(), EXC_GP);
	2024	op_bytes = !(b & 1) ? 1 : (op_bytes == 8) ? 4 : op_bytes;
	2025	if ( b & 2 )
	2026	{
	2027	/* out */
	2028	fail_if(ops->write_io == NULL);
	2029	rc = ops->write_io(port, op_bytes, _regs.eax, ctxt);
	2030
	2031	}
	2032	else
	2033	{
	2034	/* in */
	2035	dst.type = OP_REG;
	2036	dst.bytes = op_bytes;
	2037	dst.reg = (unsigned long *)&_regs.eax;
	2038	fail_if(ops->read_io == NULL);
	2039	rc = ops->read_io(port, dst.bytes, &dst.val, ctxt);
	2040	}
	2041	if ( rc != 0 )
	2042	goto done;
	2043	break;
	2044	}
	2045
	2046	case 0xe8: /* call (near) */ {
	2047	int rel = (((op_bytes == 2) && !mode_64bit())
	2048	? (int32_t)insn_fetch_type(int16_t)
	2049	: insn_fetch_type(int32_t));
	2050	op_bytes = mode_64bit() ? 8 : op_bytes;
	2051	src.val = _regs.eip;
	2052	jmp_rel(rel);
	2053	goto push;
	2054	}
	2055
	2056	case 0xe9: /* jmp (near) */ {
	2057	int rel = (((op_bytes == 2) && !mode_64bit())
	2058	? (int32_t)insn_fetch_type(int16_t)
	2059	: insn_fetch_type(int32_t));
	2060	jmp_rel(rel);
	2061	break;
	2062	}
	2063
	2064	case 0xeb: /* jmp (short) */
	2065	jmp_rel(insn_fetch_type(int8_t));
	2066	break;
	2067
	2068	case 0xf5: /* cmc */
	2069	_regs.eflags ^= EFLG_CF;
	2070	break;
	2071
	2072	case 0xf8: /* clc */
	2073	_regs.eflags &= ~EFLG_CF;
	2074	break;
	2075
	2076	case 0xf9: /* stc */
	2077	_regs.eflags \|= EFLG_CF;
	2078	break;
	2079
	2080	case 0xfa: /* cli */
	2081	generate_exception_if(!mode_iopl(), EXC_GP);
	2082	fail_if(ops->write_rflags == NULL);
	2083	if ( (rc = ops->write_rflags(_regs.eflags & ~EFLG_IF, ctxt)) != 0 )
	2084	goto done;
	2085	break;
	2086
	2087	case 0xfb: /* sti */
	2088	generate_exception_if(!mode_iopl(), EXC_GP);
	2089	fail_if(ops->write_rflags == NULL);
	2090	if ( (rc = ops->write_rflags(_regs.eflags \| EFLG_IF, ctxt)) != 0 )
	2091	goto done;
	2092	break;
	2093
	2094	case 0xfc: /* cld */
	2095	_regs.eflags &= ~EFLG_DF;
	2096	break;
	2097
	2098	case 0xfd: /* std */
	2099	_regs.eflags \|= EFLG_DF;
	2100	break;
	2101	}
	2102	goto writeback;
	2103
	2104	twobyte_insn:
	2105	switch ( b )
	2106	{
	2107	case 0x40 ... 0x4f: /* cmovcc */
	2108	dst.val = src.val;
	2109	if ( !test_cc(b, _regs.eflags) )
	2110	dst.type = OP_NONE;
	2111	break;
	2112
	2113	case 0x90 ... 0x9f: /* setcc */
	2114	dst.val = test_cc(b, _regs.eflags);
	2115	break;
	2116
	2117	case 0xb0 ... 0xb1: /* cmpxchg */
	2118	/* Save real source value, then compare EAX against destination. */
	2119	src.orig_val = src.val;
	2120	src.val = _regs.eax;
	2121	emulate_2op_SrcV("cmp", src, dst, _regs.eflags);
	2122	/* Always write back. The question is: where to? */
	2123	d \|= Mov;
	2124	if ( _regs.eflags & EFLG_ZF )
	2125	{
	2126	/* Success: write back to memory. */
	2127	dst.val = src.orig_val;
	2128	}
	2129	else
	2130	{
	2131	/* Failure: write the value we saw to EAX. */
	2132	dst.type = OP_REG;
	2133	dst.reg = (unsigned long *)&_regs.eax;
	2134	}
	2135	break;
	2136
	2137	case 0xa3: bt: /* bt */
	2138	emulate_2op_SrcV_nobyte("bt", src, dst, _regs.eflags);
	2139	break;
	2140
	2141	case 0xb3: btr: /* btr */
	2142	emulate_2op_SrcV_nobyte("btr", src, dst, _regs.eflags);
	2143	break;
	2144
	2145	case 0xab: bts: /* bts */
	2146	emulate_2op_SrcV_nobyte("bts", src, dst, _regs.eflags);
	2147	break;
	2148
	2149	case 0xaf: /* imul */
	2150	_regs.eflags &= ~(EFLG_OF\|EFLG_CF);
	2151	switch ( dst.bytes )
	2152	{
	2153	case 2:
	2154	dst.val = ((uint32_t)(int16_t)src.val *
	2155	(uint32_t)(int16_t)dst.val);
	2156	if ( (int16_t)dst.val != (uint32_t)dst.val )
	2157	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	2158	break;
	2159	#ifdef __x86_64__
	2160	case 4:
	2161	dst.val = ((uint64_t)(int32_t)src.val *
	2162	(uint64_t)(int32_t)dst.val);
	2163	if ( (int32_t)dst.val != dst.val )
	2164	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	2165	break;
	2166	#endif
	2167	default: {
	2168	unsigned long m[2] = { src.val, dst.val };
	2169	if ( imul_dbl(m) )
	2170	_regs.eflags \|= EFLG_OF\|EFLG_CF;
	2171	dst.val = m[0];
	2172	break;
	2173	}
	2174	}
	2175	break;
	2176
	2177	case 0xb6: /* movzx rm8,r{16,32,64} */
	2178	/* Recompute DstReg as we may have decoded AH/BH/CH/DH. */
	2179	dst.reg = decode_register(modrm_reg, &_regs, 0);
	2180	dst.bytes = op_bytes;
	2181	dst.val = (uint8_t)src.val;
	2182	break;
	2183
	2184	case 0xbc: /* bsf */ {
	2185	int zf;
	2186	asm ( "bsf %2,%0; setz %b1"
	2187	: "=r" (dst.val), "=q" (zf)
	2188	: "r" (src.val), "1" (0) );
	2189	_regs.eflags &= ~EFLG_ZF;
	2190	_regs.eflags \|= zf ? EFLG_ZF : 0;
	2191	break;
	2192	}
	2193
	2194	case 0xbd: /* bsr */ {
	2195	int zf;
	2196	asm ( "bsr %2,%0; setz %b1"
	2197	: "=r" (dst.val), "=q" (zf)
	2198	: "r" (src.val), "1" (0) );
	2199	_regs.eflags &= ~EFLG_ZF;
	2200	_regs.eflags \|= zf ? EFLG_ZF : 0;
	2201	break;
	2202	}
	2203
	2204	case 0xb7: /* movzx rm16,r{16,32,64} */
	2205	dst.val = (uint16_t)src.val;
	2206	break;
	2207
	2208	case 0xbb: btc: /* btc */
	2209	emulate_2op_SrcV_nobyte("btc", src, dst, _regs.eflags);
	2210	break;
	2211
	2212	case 0xba: /* Grp8 */
	2213	switch ( modrm_reg & 7 )
	2214	{
	2215	case 4: goto bt;
	2216	case 5: goto bts;
	2217	case 6: goto btr;
	2218	case 7: goto btc;
	2219	default: generate_exception_if(1, EXC_UD);
	2220	}
	2221	break;
	2222
	2223	case 0xbe: /* movsx rm8,r{16,32,64} */
	2224	/* Recompute DstReg as we may have decoded AH/BH/CH/DH. */
	2225	dst.reg = decode_register(modrm_reg, &_regs, 0);
	2226	dst.bytes = op_bytes;
	2227	dst.val = (int8_t)src.val;
	2228	break;
	2229
	2230	case 0xbf: /* movsx rm16,r{16,32,64} */
	2231	dst.val = (int16_t)src.val;
	2232	break;
	2233
	2234	case 0xc0 ... 0xc1: /* xadd */
	2235	/* Write back the register source. */
	2236	switch ( dst.bytes )
	2237	{
	2238	case 1: (uint8_t )src.reg = (uint8_t)dst.val; break;
	2239	case 2: (uint16_t )src.reg = (uint16_t)dst.val; break;
	2240	case 4: src.reg = (uint32_t)dst.val; break; / 64b reg: zero-extend */
	2241	case 8: *src.reg = dst.val; break;
	2242	}
	2243	goto add;
	2244	}
	2245	goto writeback;
	2246
	2247	twobyte_special_insn:
	2248	switch ( b )
	2249	{
	2250	case 0x06: /* clts */
	2251	generate_exception_if(!mode_ring0(), EXC_GP);
	2252	fail_if((ops->read_cr == NULL) \|\| (ops->write_cr == NULL));
	2253	if ( (rc = ops->read_cr(0, &dst.val, ctxt)) \|\|
	2254	(rc = ops->write_cr(0, dst.val&~8, ctxt)) )
	2255	goto done;
	2256	break;
	2257
	2258	case 0x08: /* invd */
	2259	case 0x09: /* wbinvd */
	2260	generate_exception_if(!mode_ring0(), EXC_GP);
	2261	fail_if(ops->wbinvd == NULL);
	2262	if ( (rc = ops->wbinvd(ctxt)) != 0 )
	2263	goto done;
	2264	break;
	2265
	2266	case 0x0d: /* GrpP (prefetch) */
	2267	case 0x18: /* Grp16 (prefetch/nop) */
	2268	case 0x19 ... 0x1f: /* nop (amd-defined) */
	2269	break;
	2270
	2271	case 0x20: /* mov cr,reg */
	2272	case 0x21: /* mov dr,reg */
	2273	case 0x22: /* mov reg,cr */
	2274	case 0x23: /* mov reg,dr */
	2275	generate_exception_if(!mode_ring0(), EXC_GP);
	2276	modrm_rm \|= (rex_prefix & 1) << 3;
	2277	modrm_reg \|= lock_prefix << 3;
	2278	if ( b & 2 )
	2279	{
	2280	/* Write to CR/DR. */
	2281	src.val = (unsigned long )decode_register(modrm_rm, &_regs, 0);
	2282	if ( !mode_64bit() )
	2283	src.val = (uint32_t)src.val;
	2284	rc = ((b & 1)
	2285	? (ops->write_dr
	2286	? ops->write_dr(modrm_reg, src.val, ctxt)
	2287	: X86EMUL_UNHANDLEABLE)
	2288	: (ops->write_cr
	2289	? ops->write_dr(modrm_reg, src.val, ctxt)
	2290	: X86EMUL_UNHANDLEABLE));
	2291	}
	2292	else
	2293	{
	2294	/* Read from CR/DR. */
	2295	dst.type = OP_REG;
	2296	dst.bytes = mode_64bit() ? 8 : 4;
	2297	dst.reg = decode_register(modrm_rm, &_regs, 0);
	2298	rc = ((b & 1)
	2299	? (ops->read_dr
	2300	? ops->read_dr(modrm_reg, &dst.val, ctxt)
	2301	: X86EMUL_UNHANDLEABLE)
	2302	: (ops->read_cr
	2303	? ops->read_dr(modrm_reg, &dst.val, ctxt)
	2304	: X86EMUL_UNHANDLEABLE));
	2305	}
	2306	if ( rc != 0 )
	2307	goto done;
	2308	break;
	2309
	2310	case 0x30: /* wrmsr */ {
	2311	uint64_t val = ((uint64_t)_regs.edx << 32) \| (uint32_t)_regs.eax;
	2312	generate_exception_if(!mode_ring0(), EXC_GP);
	2313	fail_if(ops->write_msr == NULL);
	2314	if ( (rc = ops->write_msr((uint32_t)_regs.ecx, val, ctxt)) != 0 )
	2315	goto done;
	2316	break;
	2317	}
	2318
	2319	case 0x32: /* rdmsr */ {
	2320	uint64_t val;
	2321	generate_exception_if(!mode_ring0(), EXC_GP);
	2322	fail_if(ops->read_msr == NULL);
	2323	if ( (rc = ops->read_msr((uint32_t)_regs.ecx, &val, ctxt)) != 0 )
	2324	goto done;
	2325	_regs.edx = (uint32_t)(val >> 32);
	2326	_regs.eax = (uint32_t)(val >> 0);
	2327	break;
	2328	}
	2329
	2330	case 0x80 ... 0x8f: /* jcc (near) */ {
	2331	int rel = (((op_bytes == 2) && !mode_64bit())
	2332	? (int32_t)insn_fetch_type(int16_t)
	2333	: insn_fetch_type(int32_t));
	2334	if ( test_cc(b, _regs.eflags) )
	2335	jmp_rel(rel);
	2336	break;
	2337	}
	2338
	2339	case 0xc7: /* Grp9 (cmpxchg8b) */
	2340	#if defined(__i386__)
	2341	{
	2342	unsigned long old_lo, old_hi;
	2343	generate_exception_if((modrm_reg & 7) != 1, EXC_UD);
	2344	if ( (rc = ops->read(ea.mem.seg, ea.mem.off+0, &old_lo, 4, ctxt)) \|\|
	2345	(rc = ops->read(ea.mem.seg, ea.mem.off+4, &old_hi, 4, ctxt)) )
	2346	goto done;
	2347	if ( (old_lo != _regs.eax) \|\| (old_hi != _regs.edx) )
	2348	{
	2349	_regs.eax = old_lo;
	2350	_regs.edx = old_hi;
	2351	_regs.eflags &= ~EFLG_ZF;
	2352	}
	2353	else if ( ops->cmpxchg8b == NULL )
	2354	{
	2355	rc = X86EMUL_UNHANDLEABLE;
	2356	goto done;
	2357	}
	2358	else
	2359	{
	2360	if ( (rc = ops->cmpxchg8b(ea.mem.seg, ea.mem.off, old_lo, old_hi,
	2361	_regs.ebx, _regs.ecx, ctxt)) != 0 )
	2362	goto done;
	2363	_regs.eflags \|= EFLG_ZF;
	2364	}
	2365	break;
	2366	}
	2367	#elif defined(__x86_64__)
	2368	{
	2369	unsigned long old, new;
	2370	generate_exception_if((modrm_reg & 7) != 1, EXC_UD);
	2371	if ( (rc = ops->read(ea.mem.seg, ea.mem.off, &old, 8, ctxt)) != 0 )
	2372	goto done;
	2373	if ( ((uint32_t)(old>>0) != (uint32_t)_regs.eax) \|\|
	2374	((uint32_t)(old>>32) != (uint32_t)_regs.edx) )
	2375	{
	2376	_regs.eax = (uint32_t)(old>>0);
	2377	_regs.edx = (uint32_t)(old>>32);
	2378	_regs.eflags &= ~EFLG_ZF;
	2379	}
	2380	else
	2381	{
	2382	new = (_regs.ecx<<32)\|(uint32_t)_regs.ebx;
	2383	if ( (rc = ops->cmpxchg(ea.mem.seg, ea.mem.off, old,
	2384	new, 8, ctxt)) != 0 )
	2385	goto done;
	2386	_regs.eflags \|= EFLG_ZF;
	2387	}
	2388	break;
	2389	}
	2390	#endif
	2391
	2392	case 0xc8 ... 0xcf: /* bswap */
	2393	dst.type = OP_REG;
	2394	dst.reg = decode_register(
	2395	(b & 7) \| ((rex_prefix & 1) << 3), &_regs, 0);
	2396	switch ( dst.bytes = op_bytes )
	2397	{
	2398	default: /* case 2: */
	2399	/* Undefined behaviour. Writes zero on all tested CPUs. */
	2400	dst.val = 0;
	2401	break;
	2402	case 4:
	2403	#ifdef __x86_64__
	2404	__asm__ ( "bswap %k0" : "=r" (dst.val) : "0" (*dst.reg) );
	2405	break;
	2406	case 8:
	2407	#endif
	2408	__asm__ ( "bswap %0" : "=r" (dst.val) : "0" (*dst.reg) );
	2409	break;
	2410	}
	2411	break;
	2412	}
	2413	goto writeback;
	2414
	2415	cannot_emulate:
	2416	#if 0
	2417	gdprintk(XENLOG_DEBUG, "Instr:");
	2418	for ( ea.mem.off = ctxt->regs->eip; ea.mem.off < _regs.eip; ea.mem.off++ )
	2419	{
	2420	unsigned long x;
	2421	ops->insn_fetch(x86_seg_cs, ea.mem.off, &x, 1, ctxt);
	2422	printk(" %02x", (uint8_t)x);
	2423	}
	2424	printk("\n");
	2425	#endif
	2426	return X86EMUL_UNHANDLEABLE;
	2427	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: