emulate.c 108 KB
Newer Older
1
/******************************************************************************
2
 * emulate.c
3 4 5 6 7 8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
10 11
 *
 * Copyright (C) 2006 Qumranet
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
13 14 15 16 17 18 19 20 21 22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27

28
#include "x86.h"
29
#include "tss.h"
30

31 32 33
/*
 * Operand types
 */
34 35 36 37 38 39 40 41 42
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
43 44 45 46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
#define OpImm             12ull  /* Sign extended immediate */
47 48 49 50 51 52 53
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
54 55 56 57 58 59
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
60 61

#define OpBits             5  /* Width of operand field */
62
#define OpMask             ((1ull << OpBits) - 1)
63

64 65 66 67 68 69 70 71 72 73
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
74
#define ByteOp      (1<<0)	/* 8-bit operands. */
75
/* Destination operand type. */
76 77 78 79 80 81 82 83 84 85
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
86
/* Source operand type. */
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
#define SrcDX       (OpDX << SrcShift)
#define SrcMask     (OpMask << SrcShift)
105 106 107 108 109 110 111 112 113 114
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
#define Sse         (1<<18)     /* SSE Vector instruction */
115 116 117 118
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
119
/* Misc flags */
120
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
121
#define VendorSpecific (1<<22) /* Vendor specific instruction */
122
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124
#define Undefined   (1<<25) /* No Such Instruction */
125
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
126
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
127
#define No64	    (1<<28)
128
#define PageTable   (1 << 29)   /* instruction used to write page table */
129
/* Source 2 operand type */
130
#define Src2Shift   (30)
131 132 133 134 135
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
136 137 138 139 140 141
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
142
#define Src2Mask    (OpMask << Src2Shift)
143

144 145 146 147 148 149 150 151
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
152

153
struct opcode {
154 155
	u64 flags : 56;
	u64 intercept : 8;
156
	union {
157
		int (*execute)(struct x86_emulate_ctxt *ctxt);
158 159
		struct opcode *group;
		struct group_dual *gdual;
160
		struct gprefix *gprefix;
161
	} u;
162
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
163 164 165 166 167
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
168 169
};

170 171 172 173 174 175 176
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

177
/* EFLAGS bit definitions. */
178 179 180 181
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
182 183
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
184 185
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
186 187
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
188
#define EFLG_IF (1<<9)
189
#define EFLG_TF (1<<8)
190 191 192 193 194 195
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

196 197 198
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

199 200 201 202 203 204 205
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

206
#if defined(CONFIG_X86_64)
207 208 209 210 211 212 213 214 215 216 217 218 219 220
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
236 237 238 239 240 241 242 243 244

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

245 246 247 248 249 250
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

251
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
252 253 254 255 256
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
257 258
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
259
			  "=&r" (_tmp)					\
260
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
261
	} while (0)
262 263


264
/* Raw emulation: instruction has two explicit operands. */
265
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
266 267 268
	do {								\
		unsigned long _tmp;					\
									\
269
		switch ((ctxt)->dst.bytes) {				\
270
		case 2:							\
271
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
272 273
			break;						\
		case 4:							\
274
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
275 276
			break;						\
		case 8:							\
277
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
278 279
			break;						\
		}							\
280 281
	} while (0)

282
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
283
	do {								     \
284
		unsigned long _tmp;					     \
285
		switch ((ctxt)->dst.bytes) {				     \
286
		case 1:							     \
287
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
288 289
			break;						     \
		default:						     \
290
			__emulate_2op_nobyte(ctxt, _op,			     \
291 292 293 294 295 296
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
297 298
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
299 300

/* Source operand is byte, word, long or quad sized. */
301 302
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
303 304

/* Source operand is word, long or quad sized. */
305 306
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
307

308
/* Instruction has three operands and one operand is stored in ECX register */
309
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
310 311
	do {								\
		unsigned long _tmp;					\
312 313 314
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
315 316 317 318 319
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
320
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
321 322 323
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
324 325 326
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
327 328
	} while (0)

329
#define emulate_2op_cl(ctxt, _op)					\
330
	do {								\
331
		switch ((ctxt)->dst.bytes) {				\
332
		case 2:							\
333
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
334 335
			break;						\
		case 4:							\
336
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
337 338
			break;						\
		case 8:							\
339
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
340 341
			break;						\
		}							\
342 343
	} while (0)

344
#define __emulate_1op(ctxt, _op, _suffix)				\
345 346 347
	do {								\
		unsigned long _tmp;					\
									\
348 349 350 351
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
352
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353 354 355 356 357
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
358
#define emulate_1op(ctxt, _op)						\
359
	do {								\
360 361 362 363 364
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
365 366 367
		}							\
	} while (0)

368
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
369 370
	do {								\
		unsigned long _tmp;					\
371 372
		ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];		\
		ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];		\
373 374 375 376 377 378 379 380 381 382 383 384
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
385 386 387 388
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val),	\
			  "a" (*rax), "d" (*rdx));			\
389 390
	} while (0)

391
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
392
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
393
	do {								\
394
		switch((ctxt)->src.bytes) {				\
395
		case 1:							\
396
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
397 398
			break;						\
		case 2:							\
399
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
400 401
			break;						\
		case 4:							\
402
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
403 404
			break;						\
		case 8: ON64(						\
405
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
406 407 408 409
			break;						\
		}							\
	} while (0)

410 411 412 413 414 415
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
416 417 418 419 420 421 422 423
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
424 425 426
		.next_rip   = ctxt->eip,
	};

427
	return ctxt->ops->intercept(ctxt, &info, stage);
428 429
}

430
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431
{
432
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
433 434
}

435
/* Access/update address held in a register, based on addressing mode. */
436
static inline unsigned long
437
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438
{
439
	if (ctxt->ad_bytes == sizeof(unsigned long))
440 441
		return reg;
	else
442
		return reg & ad_mask(ctxt);
443 444 445
}

static inline unsigned long
446
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447
{
448
	return address_mask(ctxt, reg);
449 450
}

451
static inline void
452
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453
{
454
	if (ctxt->ad_bytes == sizeof(unsigned long))
455 456
		*reg += inc;
	else
457
		*reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
458
}
459

460
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
461
{
462
	register_address_increment(ctxt, &ctxt->_eip, rel);
463
}
464

465 466 467 468 469 470 471
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

472
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
473
{
474 475
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
476 477
}

478
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
479 480 481 482
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

483
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
484 485
}

486
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
487
{
488
	if (!ctxt->has_seg_override)
489 490
		return 0;

491
	return ctxt->seg_override;
492 493
}

494 495
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
496
{
497 498 499
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
500
	return X86EMUL_PROPAGATE_FAULT;
501 502
}

503 504 505 506 507
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

508
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
509
{
510
	return emulate_exception(ctxt, GP_VECTOR, err, true);
511 512
}

513 514 515 516 517
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

518
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
519
{
520
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
521 522
}

523
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
524
{
525
	return emulate_exception(ctxt, TS_VECTOR, err, true);
526 527
}

528 529
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
530
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
531 532
}

Avi Kivity's avatar
Avi Kivity committed
533 534 535 536 537
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

558
static int __linearize(struct x86_emulate_ctxt *ctxt,
559
		     struct segmented_address addr,
560
		     unsigned size, bool write, bool fetch,
561 562
		     ulong *linear)
{
563 564
	struct desc_struct desc;
	bool usable;
565
	ulong la;
566
	u32 lim;
567
	u16 sel;
568
	unsigned cpl, rpl;
569

570
	la = seg_base(ctxt, addr.seg) + addr.ea;
571 572 573 574 575 576 577 578
	switch (ctxt->mode) {
	case X86EMUL_MODE_REAL:
		break;
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
579 580
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
581 582 583 584 585 586
		if (!usable)
			goto bad;
		/* code segment or read-only data segment */
		if (((desc.type & 8) || !(desc.type & 2)) && write)
			goto bad;
		/* unreadable code segment */
587
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
588 589 590 591 592 593 594 595 596 597 598 599 600 601
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
			/* exapand-down segment */
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
602
		cpl = ctxt->ops->cpl(ctxt);
603
		rpl = sel & 3;
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
		cpl = max(cpl, rpl);
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
620
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
621 622 623
		la &= (u32)-1;
	*linear = la;
	return X86EMUL_CONTINUE;
624 625 626 627 628
bad:
	if (addr.seg == VCPU_SREG_SS)
		return emulate_ss(ctxt, addr.seg);
	else
		return emulate_gp(ctxt, addr.seg);
629 630
}

631 632 633 634 635 636 637 638 639
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


640 641 642 643 644
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
645 646 647
	int rc;
	ulong linear;

648
	rc = linearize(ctxt, addr, size, false, &linear);
649 650
	if (rc != X86EMUL_CONTINUE)
		return rc;
651
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
652 653
}

654 655 656 657 658 659 660 661
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
662
{
663
	struct fetch_cache *fc = &ctxt->fetch;
664
	int rc;
665
	int size, cur_size;
666

667
	if (ctxt->_eip == fc->end) {
668
		unsigned long linear;
669 670
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
671
		cur_size = fc->end - fc->start;
672 673
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
674
		rc = __linearize(ctxt, addr, size, false, true, &linear);
675
		if (unlikely(rc != X86EMUL_CONTINUE))
676
			return rc;
677 678
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
679
		if (unlikely(rc != X86EMUL_CONTINUE))
680
			return rc;
681
		fc->end += size;
682
	}
683 684
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
685
	return X86EMUL_CONTINUE;
686 687 688
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689
			 void *dest, unsigned size)
690
{
691
	int rc;
692

693
	/* x86 instructions are limited to 15 bytes. */
694
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
695
		return X86EMUL_UNHANDLEABLE;
696
	while (size--) {
697
		rc = do_insn_fetch_byte(ctxt, dest++);
698
		if (rc != X86EMUL_CONTINUE)
699 700
			return rc;
	}
701
	return X86EMUL_CONTINUE;
702 703
}

704
/* Fetch next part of the instruction being emulated. */
705
#define insn_fetch(_type, _ctxt)					\
706
({	unsigned long _x;						\
707
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
708 709 710 711 712
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

713 714
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
715 716 717 718
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

719 720 721 722 723 724 725
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
static void *decode_register(u8 modrm_reg, unsigned long *regs,
			     int highbyte_regs)
726 727 728 729 730 731 732 733 734 735
{
	void *p;

	p = &regs[modrm_reg];
	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736
			   struct segmented_address addr,
737 738 739 740 741 742 743
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
744
	rc = segmented_read_std(ctxt, addr, size, 2);
745
	if (rc != X86EMUL_CONTINUE)
746
		return rc;
747
	addr.ea += 2;
748
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
749 750 751
	return rc;
}

752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
static int test_cc(unsigned int condition, unsigned int flags)
{
	int rc = 0;

	switch ((condition & 15) >> 1) {
	case 0: /* o */
		rc |= (flags & EFLG_OF);
		break;
	case 1: /* b/c/nae */
		rc |= (flags & EFLG_CF);
		break;
	case 2: /* z/e */
		rc |= (flags & EFLG_ZF);
		break;
	case 3: /* be/na */
		rc |= (flags & (EFLG_CF|EFLG_ZF));
		break;
	case 4: /* s */
		rc |= (flags & EFLG_SF);
		break;
	case 5: /* p/pe */
		rc |= (flags & EFLG_PF);
		break;
	case 7: /* le/ng */
		rc |= (flags & EFLG_ZF);
		/* fall through */
	case 6: /* l/nge */
		rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
		break;
	}

	/* Odd condition identifiers (lsb == 1) have inverted sense. */
	return (!!rc ^ (condition & 1));
}

787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

Avi Kivity's avatar
Avi Kivity committed
805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
#ifdef CONFIG_X86_64
	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
				    struct operand *op,
862 863
				    int inhibit_bytereg)
{
864 865
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
866

867 868
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
Avi Kivity's avatar
Avi Kivity committed
869

870
	if (ctxt->d & Sse) {
Avi Kivity's avatar
Avi Kivity committed
871 872 873 874 875 876 877
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}

878
	op->type = OP_REG;
879 880
	if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
		op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
881 882
		op->bytes = 1;
	} else {
883 884
		op->addr.reg = decode_register(reg, ctxt->regs, 0);
		op->bytes = ctxt->op_bytes;
885
	}
886
	fetch_register_operand(op);
887 888 889
	op->orig_val = op->val;
}

890
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
891
			struct operand *op)
892 893
{
	u8 sib;
894
	int index_reg = 0, base_reg = 0, scale;
895
	int rc = X86EMUL_CONTINUE;
896
	ulong modrm_ea = 0;
897

898 899 900 901
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
902 903
	}

904
	ctxt->modrm = insn_fetch(u8, ctxt);
905 906 907 908
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
909

910
	if (ctxt->modrm_mod == 3) {
911
		op->type = OP_REG;
912 913 914 915
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = decode_register(ctxt->modrm_rm,
					       ctxt->regs, ctxt->d & ByteOp);
		if (ctxt->d & Sse) {
Avi Kivity's avatar
Avi Kivity committed
916 917
			op->type = OP_XMM;
			op->bytes = 16;
918 919
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
Avi Kivity's avatar
Avi Kivity committed
920 921
			return rc;
		}
922
		fetch_register_operand(op);
923 924 925
		return rc;
	}

926 927
	op->type = OP_MEM;

928 929 930 931 932
	if (ctxt->ad_bytes == 2) {
		unsigned bx = ctxt->regs[VCPU_REGS_RBX];
		unsigned bp = ctxt->regs[VCPU_REGS_RBP];
		unsigned si = ctxt->regs[VCPU_REGS_RSI];
		unsigned di = ctxt->regs[VCPU_REGS_RDI];
933 934

		/* 16-bit ModR/M decode. */
935
		switch (ctxt->modrm_mod) {
936
		case 0:
937
			if (ctxt->modrm_rm == 6)
938
				modrm_ea += insn_fetch(u16, ctxt);
939 940
			break;
		case 1:
941
			modrm_ea += insn_fetch(s8, ctxt);
942 943
			break;
		case 2:
944
			modrm_ea += insn_fetch(u16, ctxt);
945 946
			break;
		}
947
		switch (ctxt->modrm_rm) {
948
		case 0:
949
			modrm_ea += bx + si;
950 951
			break;
		case 1:
952
			modrm_ea += bx + di;
953 954
			break;
		case 2:
955
			modrm_ea += bp + si;
956 957
			break;
		case 3:
958
			modrm_ea += bp + di;
959 960
			break;
		case 4:
961
			modrm_ea += si;
962 963
			break;
		case 5:
964
			modrm_ea += di;
965 966
			break;
		case 6:
967
			if (ctxt->modrm_mod != 0)
968
				modrm_ea += bp;
969 970
			break;
		case 7:
971
			modrm_ea += bx;
972 973
			break;
		}
974 975 976
		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
			ctxt->modrm_seg = VCPU_SREG_SS;
977
		modrm_ea = (u16)modrm_ea;
978 979
	} else {
		/* 32/64-bit ModR/M decode. */
980
		if ((ctxt->modrm_rm & 7) == 4) {
981
			sib = insn_fetch(u8, ctxt);
982 983 984 985
			index_reg |= (sib >> 3) & 7;
			base_reg |= sib & 7;
			scale = sib >> 6;

986
			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987
				modrm_ea += insn_fetch(s32, ctxt);
988
			else
989
				modrm_ea += ctxt->regs[base_reg];
990
			if (index_reg != 4)
991 992
				modrm_ea += ctxt->regs[index_reg] << scale;
		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993
			if (ctxt->mode == X86EMUL_MODE_PROT64)
994
				ctxt->rip_relative = 1;
995
		} else
996 997
			modrm_ea += ctxt->regs[ctxt->modrm_rm];
		switch (ctxt->modrm_mod) {
998
		case 0:
999
			if (ctxt->modrm_rm == 5)
1000
				modrm_ea += insn_fetch(s32, ctxt);
1001 1002
			break;
		case 1:
1003
			modrm_ea += insn_fetch(s8, ctxt);
1004 1005
			break;
		case 2:
1006
			modrm_ea += insn_fetch(s32, ctxt);
1007 1008 1009
			break;
		}
	}
1010
	op->addr.mem.ea = modrm_ea;
1011 1012 1013 1014 1015
done:
	return rc;
}

static int decode_abs(struct x86_emulate_ctxt *ctxt,
1016
		      struct operand *op)
1017
{
1018
	int rc = X86EMUL_CONTINUE;
1019

1020
	op->type = OP_MEM;
1021
	switch (ctxt->ad_bytes) {
1022
	case 2:
1023
		op->addr.mem.ea = insn_fetch(u16, ctxt);
1024 1025
		break;
	case 4:
1026
		op->addr.mem.ea = insn_fetch(u32, ctxt);
1027 1028
		break;
	case 8:
1029
		op->addr.mem.ea = insn_fetch(u64, ctxt);
1030 1031 1032 1033 1034 1035
		break;
	}
done:
	return rc;
}

1036
static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1037
{
1038
	long sv = 0, mask;
1039

1040 1041
	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
		mask = ~(ctxt->dst.bytes * 8 - 1);
1042

1043 1044 1045 1046
		if (ctxt->src.bytes == 2)
			sv = (s16)ctxt->src.val & (s16)mask;
		else if (ctxt->src.bytes == 4)
			sv = (s32)ctxt->src.val & (s32)mask;
1047

1048
		ctxt->dst.addr.mem.ea += (sv >> 3);
1049
	}
1050 1051

	/* only subword offset */
1052
	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1053 1054
}

1055 1056
static int read_emulated(struct x86_emulate_ctxt *ctxt,
			 unsigned long addr, void *dest, unsigned size)
1057
{
1058
	int rc;
1059
	struct read_cache *mc = &ctxt->mem_read;
1060

1061 1062 1063 1064 1065
	while (size) {
		int n = min(size, 8u);
		size -= n;
		if (mc->pos < mc->end)
			goto read_cached;
1066

1067 1068
		rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
					      &ctxt->exception);
1069 1070 1071
		if (rc != X86EMUL_CONTINUE)
			return rc;
		mc->end += n;
1072

1073 1074 1075 1076 1077
	read_cached:
		memcpy(dest, mc->data + mc->pos, n);
		mc->pos += n;
		dest += n;
		addr += n;
1078
	}
1079 1080
	return X86EMUL_CONTINUE;
}
1081

1082 1083 1084 1085 1086
static int segmented_read(struct x86_emulate_ctxt *ctxt,
			  struct segmented_address addr,
			  void *data,
			  unsigned size)
{
1087 1088 1089
	int rc;
	ulong linear;

1090
	rc = linearize(ctxt, addr, size, false, &linear);
1091 1092
	if (rc != X86EMUL_CONTINUE)
		return rc;
1093
	return read_emulated(ctxt, linear, data, size);
1094 1095 1096 1097 1098 1099 1100
}

static int segmented_write(struct x86_emulate_ctxt *ctxt,
			   struct segmented_address addr,
			   const void *data,
			   unsigned size)
{
1101 1102 1103
	int rc;
	ulong linear;

1104
	rc = linearize(ctxt, addr, size, true, &linear);
1105 1106
	if (rc != X86EMUL_CONTINUE)
		return rc;
1107 1108
	return ctxt->ops->write_emulated(ctxt, linear, data, size,
					 &ctxt->exception);
1109 1110 1111 1112 1113 1114 1115
}

static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
			     struct segmented_address addr,
			     const void *orig_data, const void *data,
			     unsigned size)
{
1116 1117 1118
	int rc;
	ulong linear;

1119
	rc = linearize(ctxt, addr, size, true, &linear);
1120 1121
	if (rc != X86EMUL_CONTINUE)
		return rc;
1122 1123
	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
					   size, &ctxt->exception);
1124 1125
}

1126 1127 1128 1129
static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
			   unsigned int size, unsigned short port,
			   void *dest)
{
1130
	struct read_cache *rc = &ctxt->io_read;
1131

1132 1133
	if (rc->pos == rc->end) { /* refill pio read ahead */
		unsigned int in_page, n;
1134 1135
		unsigned int count = ctxt->rep_prefix ?
			address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136
		in_page = (ctxt->eflags & EFLG_DF) ?
1137 1138
			offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
			PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 1140 1141 1142 1143
		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
			count);
		if (n == 0)
			n = 1;
		rc->pos = rc->end = 0;
1144
		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1145 1146
			return 0;
		rc->end = n * size;
1147 1148
	}

1149 1150 1151 1152
	memcpy(dest, rc->data + rc->pos, size);
	rc->pos += size;
	return 1;
}
1153

1154 1155 1156
static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
				     u16 selector, struct desc_ptr *dt)
{
1157 1158
	struct x86_emulate_ops *ops = ctxt->ops;

1159 1160
	if (selector & 1 << 2) {
		struct desc_struct desc;
1161 1162
		u16 sel;

1163
		memset (dt, 0, sizeof *dt);
1164
		if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1165
			return;
1166

1167 1168 1169
		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
		dt->address = get_desc_base(&desc);
	} else
1170
		ops->get_gdt(ctxt, dt);
1171
}
1172

1173 1174 1175 1176 1177 1178 1179
/* allowed just for 8 bytes segments */
static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1180

1181
	get_descriptor_table_ptr(ctxt, selector, &dt);
1182

1183 1184
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1185

1186 1187 1188
	addr = dt.address + index * 8;
	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
				   &ctxt->exception);
1189
}
1190

1191 1192 1193 1194 1195 1196 1197
/* allowed just for 8 bytes segments */
static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				    u16 selector, struct desc_struct *desc)
{
	struct desc_ptr dt;
	u16 index = selector >> 3;
	ulong addr;
1198

1199
	get_descriptor_table_ptr(ctxt, selector, &dt);
1200

1201 1202
	if (dt.size < index * 8 + 7)
		return emulate_gp(ctxt, selector & 0xfffc);
1203

1204
	addr = dt.address + index * 8;
1205 1206
	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
				    &ctxt->exception);
1207
}
1208

1209
/* Does not support long mode */
1210 1211 1212 1213 1214 1215 1216 1217 1218
static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
				   u16 selector, int seg)
{
	struct desc_struct seg_desc;
	u8 dpl, rpl, cpl;
	unsigned err_vec = GP_VECTOR;
	u32 err_code = 0;
	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
	int ret;
1219

1220
	memset(&seg_desc, 0, sizeof seg_desc);
1221

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
	    || ctxt->mode == X86EMUL_MODE_REAL) {
		/* set real mode segment descriptor */
		set_desc_base(&seg_desc, selector << 4);
		set_desc_limit(&seg_desc, 0xffff);
		seg_desc.type = 3;
		seg_desc.p = 1;
		seg_desc.s = 1;
		goto load;
	}

	/* NULL selector is not valid for TR, CS and SS */
	if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
	    && null_selector)
		goto exception;

	/* TR should be in GDT only */
	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
		goto exception;

	if (null_selector) /* for NULL selector skip all following checks */
		goto load;

1245
	ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
	if (ret != X86EMUL_CONTINUE)
		return ret;

	err_code = selector & 0xfffc;
	err_vec = GP_VECTOR;

	/* can't load system descriptor into segment selecor */
	if (seg <= VCPU_SREG_GS && !seg_desc.s)
		goto exception;

	if (!seg_desc.p) {
		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
		goto exception;
	}

	rpl = selector & 3;
	dpl = seg_desc.dpl;
1263
	cpl = ctxt->ops->cpl(ctxt);
1264 1265 1266 1267 1268 1269 1270 1271 1272

	switch (seg) {
	case VCPU_SREG_SS:
		/*
		 * segment is not a writable data segment or segment
		 * selector's RPL != CPL or segment selector's RPL != CPL
		 */
		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
			goto exception;
1273
		break;
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	case VCPU_SREG_CS:
		if (!(seg_desc.type & 8))
			goto exception;

		if (seg_desc.type & 4) {
			/* conforming */
			if (dpl > cpl)
				goto exception;
		} else {
			/* nonconforming */
			if (rpl > cpl || dpl != cpl)
				goto exception;
		}
		/* CS(RPL) <- CPL */
		selector = (selector & 0xfffc) | cpl;
1289
		break;
1290 1291 1292 1293 1294 1295 1296 1297 1298
	case VCPU_SREG_TR:
		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
			goto exception;
		break;
	case VCPU_SREG_LDTR:
		if (seg_desc.s || seg_desc.type != 2)
			goto exception;
		break;
	default: /*  DS, ES, FS, or GS */
1299
		/*
1300 1301 1302
		 * segment is not a data or readable code segment or
		 * ((segment is a data or nonconforming code segment)
		 * and (both RPL and CPL > DPL))
1303
		 */
1304 1305 1306 1307
		if ((seg_desc.type & 0xa) == 0x8 ||
		    (((seg_desc.type & 0xc) != 0xc) &&
		     (rpl > dpl && cpl > dpl)))
			goto exception;
1308
		break;
1309 1310 1311 1312 1313
	}

	if (seg_desc.s) {
		/* mark segment as accessed */
		seg_desc.type |= 1;
1314
		ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 1316 1317 1318
		if (ret != X86EMUL_CONTINUE)
			return ret;
	}
load:
1319
	ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 1321 1322 1323 1324 1325
	return X86EMUL_CONTINUE;
exception:
	emulate_exception(ctxt, err_vec, err_code, true);
	return X86EMUL_PROPAGATE_FAULT;
}

1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
static void write_register_operand(struct operand *op)
{
	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
	switch (op->bytes) {
	case 1:
		*(u8 *)op->addr.reg = (u8)op->val;
		break;
	case 2:
		*(u16 *)op->addr.reg = (u16)op->val;
		break;
	case 4:
		*op->addr.reg = (u32)op->val;
		break;	/* 64b: zero-extend */
	case 8:
		*op->addr.reg = op->val;
		break;
	}
}

1345
static int writeback(struct x86_emulate_ctxt *ctxt)
1346 1347 1348
{
	int rc;

1349
	switch (ctxt->dst.type) {
1350
	case OP_REG:
1351
		write_register_operand(&ctxt->dst);
1352
		break;
1353
	case OP_MEM:
1354
		if (ctxt->lock_prefix)
1355
			rc = segmented_cmpxchg(ctxt,
1356 1357 1358 1359
					       ctxt->dst.addr.mem,
					       &ctxt->dst.orig_val,
					       &ctxt->dst.val,
					       ctxt->dst.bytes);
1360
		else
1361
			rc = segmented_write(ctxt,
1362 1363 1364
					     ctxt->dst.addr.mem,
					     &ctxt->dst.val,
					     ctxt->dst.bytes);
1365 1366
		if (rc != X86EMUL_CONTINUE)
			return rc;
1367
		break;
Avi Kivity's avatar
Avi Kivity committed
1368
	case OP_XMM:
1369
		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
Avi Kivity's avatar
Avi Kivity committed
1370
		break;
1371 1372
	case OP_NONE:
		/* no writeback */
1373
		break;
1374
	default:
1375
		break;
1376
	}
1377 1378
	return X86EMUL_CONTINUE;
}
1379

1380
static int em_push(struct x86_emulate_ctxt *ctxt)
1381
{
1382
	struct segmented_address addr;
1383

1384 1385
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 1387 1388
	addr.seg = VCPU_SREG_SS;

	/* Disable writeback. */
1389 1390
	ctxt->dst.type = OP_NONE;
	return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1391
}
1392

1393 1394 1395 1396
static int emulate_pop(struct x86_emulate_ctxt *ctxt,
		       void *dest, int len)
{
	int rc;
1397
	struct segmented_address addr;
1398

1399
	addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400
	addr.seg = VCPU_SREG_SS;
1401
	rc = segmented_read(ctxt, addr, dest, len);
1402 1403 1404
	if (rc != X86EMUL_CONTINUE)
		return rc;

1405
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1406
	return rc;
1407 1408
}

1409 1410
static int em_pop(struct x86_emulate_ctxt *ctxt)
{
1411
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1412 1413
}

1414
static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415
			void *dest, int len)
1416 1417
{
	int rc;
1418 1419
	unsigned long val, change_mask;
	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420
	int cpl = ctxt->ops->cpl(ctxt);
1421

1422
	rc = emulate_pop(ctxt, &val, len);
1423 1424
	if (rc != X86EMUL_CONTINUE)
		return rc;
1425

1426 1427
	change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
		| EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1428

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
	switch(ctxt->mode) {
	case X86EMUL_MODE_PROT64:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT16:
		if (cpl == 0)
			change_mask |= EFLG_IOPL;
		if (cpl <= iopl)
			change_mask |= EFLG_IF;
		break;
	case X86EMUL_MODE_VM86:
1439 1440
		if (iopl < 3)
			return emulate_gp(ctxt, 0);
1441 1442 1443 1444 1445
		change_mask |= EFLG_IF;
		break;
	default: /* real mode */
		change_mask |= (EFLG_IOPL | EFLG_IF);
		break;
1446
	}
1447 1448 1449 1450 1451

	*(unsigned long *)dest =
		(ctxt->eflags & ~change_mask) | (val & change_mask);

	return rc;
1452 1453
}

1454 1455
static int em_popf(struct x86_emulate_ctxt *ctxt)
{
1456 1457 1458 1459
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->eflags;
	ctxt->dst.bytes = ctxt->op_bytes;
	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1460 1461
}

1462
static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1463
{
1464 1465
	int seg = ctxt->src2.val;

1466
	ctxt->src.val = get_segment_selector(ctxt, seg);
1467

1468
	return em_push(ctxt);
1469 1470
}

1471
static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1472
{
1473
	int seg = ctxt->src2.val;
1474 1475
	unsigned long selector;
	int rc;
1476

1477
	rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1478 1479 1480
	if (rc != X86EMUL_CONTINUE)
		return rc;

1481
	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1482
	return rc;
1483 1484
}

1485
static int em_pusha(struct x86_emulate_ctxt *ctxt)
1486
{
1487
	unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1488 1489
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RAX;
1490

1491 1492
	while (reg <= VCPU_REGS_RDI) {
		(reg == VCPU_REGS_RSP) ?
1493
		(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1494

1495
		rc = em_push(ctxt);
1496 1497
		if (rc != X86EMUL_CONTINUE)
			return rc;
1498

1499
		++reg;
1500 1501
	}

1502
	return rc;
1503 1504
}

1505 1506
static int em_pushf(struct x86_emulate_ctxt *ctxt)
{
1507
	ctxt->src.val =  (unsigned long)ctxt->eflags;
1508 1509 1510
	return em_push(ctxt);
}

1511
static int em_popa(struct x86_emulate_ctxt *ctxt)
1512
{
1513 1514
	int rc = X86EMUL_CONTINUE;
	int reg = VCPU_REGS_RDI;
1515

1516 1517
	while (reg >= VCPU_REGS_RAX) {
		if (reg == VCPU_REGS_RSP) {
1518 1519
			register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
							ctxt->op_bytes);
1520 1521
			--reg;
		}
1522

1523
		rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1524 1525 1526
		if (rc != X86EMUL_CONTINUE)
			break;
		--reg;
1527
	}
1528
	return rc;
1529 1530
}

1531
int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1532
{
1533
	struct x86_emulate_ops *ops = ctxt->ops;
1534
	int rc;
1535 1536 1537 1538 1539 1540
	struct desc_ptr dt;
	gva_t cs_addr;
	gva_t eip_addr;
	u16 cs, eip;

	/* TODO: Add limit checks */
1541
	ctxt->src.val = ctxt->eflags;
1542
	rc = em_push(ctxt);
1543 1544
	if (rc != X86EMUL_CONTINUE)
		return rc;
1545 1546 1547

	ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);

1548
	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1549
	rc = em_push(ctxt);
1550 1551
	if (rc != X86EMUL_CONTINUE)
		return rc;
1552

1553
	ctxt->src.val = ctxt->_eip;
1554
	rc = em_push(ctxt);
1555 1556 1557
	if (rc != X86EMUL_CONTINUE)
		return rc;

1558
	ops->get_idt(ctxt, &dt);
1559 1560 1561 1562

	eip_addr = dt.address + (irq << 2);
	cs_addr = dt.address + (irq << 2) + 2;

1563
	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1564 1565 1566
	if (rc != X86EMUL_CONTINUE)
		return rc;

1567
	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1568 1569 1570
	if (rc != X86EMUL_CONTINUE)
		return rc;

1571
	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1572 1573 1574
	if (rc != X86EMUL_CONTINUE)
		return rc;

1575
	ctxt->_eip = eip;
1576 1577 1578 1579

	return rc;
}

1580
static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1581 1582 1583
{
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1584
		return emulate_int_real(ctxt, irq);
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
	default:
		/* Protected mode interrupts unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
	}
}

1595
static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1596
{
1597 1598 1599 1600 1601 1602 1603 1604
	int rc = X86EMUL_CONTINUE;
	unsigned long temp_eip = 0;
	unsigned long temp_eflags = 0;
	unsigned long cs = 0;
	unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
			     EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
			     EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
	unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1605

1606
	/* TODO: Add stack limit check */
1607

1608
	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1609

1610 1611
	if (rc != X86EMUL_CONTINUE)
		return rc;
1612

1613 1614
	if (temp_eip & ~0xffff)
		return emulate_gp(ctxt, 0);
1615

1616
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1617

1618 1619
	if (rc != X86EMUL_CONTINUE)
		return rc;
1620

1621
	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1622

1623 1624
	if (rc != X86EMUL_CONTINUE)
		return rc;
1625

1626
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1627

1628 1629
	if (rc != X86EMUL_CONTINUE)
		return rc;
1630

1631
	ctxt->_eip = temp_eip;
1632 1633


1634
	if (ctxt->op_bytes == 4)
1635
		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1636
	else if (ctxt->op_bytes == 2) {
1637 1638
		ctxt->eflags &= ~0xffff;
		ctxt->eflags |= temp_eflags;
1639
	}
1640 1641 1642 1643 1644

	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
	ctxt->eflags |= EFLG_RESERVED_ONE_MASK;

	return rc;
1645 1646
}

1647
static int em_iret(struct x86_emulate_ctxt *ctxt)
1648
{
1649 1650
	switch(ctxt->mode) {
	case X86EMUL_MODE_REAL:
1651
		return emulate_iret_real(ctxt);
1652 1653 1654 1655
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
	case X86EMUL_MODE_PROT32:
	case X86EMUL_MODE_PROT64:
1656
	default:
1657 1658
		/* iret from protected mode unimplemented yet */
		return X86EMUL_UNHANDLEABLE;
1659 1660 1661
	}
}

1662 1663 1664 1665 1666
static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
{
	int rc;
	unsigned short sel;

1667
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1668

1669
	rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1670 1671 1672
	if (rc != X86EMUL_CONTINUE)
		return rc;

1673 1674
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1675 1676 1677
	return X86EMUL_CONTINUE;
}

1678
static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1679
{
1680
	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1681 1682
}

1683
static int em_grp2(struct x86_emulate_ctxt *ctxt)
1684
{
1685
	switch (ctxt->modrm_reg) {
1686
	case 0:	/* rol */
1687
		emulate_2op_SrcB(ctxt, "rol");
1688 1689
		break;
	case 1:	/* ror */
1690
		emulate_2op_SrcB(ctxt, "ror");
1691 1692
		break;
	case 2:	/* rcl */
1693
		emulate_2op_SrcB(ctxt, "rcl");
1694 1695
		break;
	case 3:	/* rcr */
1696
		emulate_2op_SrcB(ctxt, "rcr");
1697 1698 1699
		break;
	case 4:	/* sal/shl */
	case 6:	/* sal/shl */
1700
		emulate_2op_SrcB(ctxt, "sal");
1701 1702
		break;
	case 5:	/* shr */
1703
		emulate_2op_SrcB(ctxt, "shr");
1704 1705
		break;
	case 7:	/* sar */
1706
		emulate_2op_SrcB(ctxt, "sar");
1707 1708
		break;
	}
1709
	return X86EMUL_CONTINUE;
1710 1711
}

1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
static int em_not(struct x86_emulate_ctxt *ctxt)
{
	ctxt->dst.val = ~ctxt->dst.val;
	return X86EMUL_CONTINUE;
}

static int em_neg(struct x86_emulate_ctxt *ctxt)
{
	emulate_1op(ctxt, "neg");
	return X86EMUL_CONTINUE;
}

static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "mul", ex);
	return X86EMUL_CONTINUE;
}

static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 ex = 0;

	emulate_1op_rax_rdx(ctxt, "imul", ex);
	return X86EMUL_CONTINUE;
}

static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1741
{
1742
	u8 de = 0;
1743

1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
	emulate_1op_rax_rdx(ctxt, "div", de);
	if (de)
		return emulate_de(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
{
	u8 de = 0;

	emulate_1op_rax_rdx(ctxt, "idiv", de);
1755 1756
	if (de)
		return emulate_de(ctxt);
1757
	return X86EMUL_CONTINUE;
1758 1759
}

1760
static int em_grp45(struct x86_emulate_ctxt *ctxt)
1761
{
1762
	int rc = X86EMUL_CONTINUE;
1763

1764
	switch (ctxt->modrm_reg) {
1765
	case 0:	/* inc */
1766
		emulate_1op(ctxt, "inc");
1767 1768
		break;
	case 1:	/* dec */
1769
		emulate_1op(ctxt, "dec");
1770
		break;
1771 1772
	case 2: /* call near abs */ {
		long int old_eip;
1773 1774 1775
		old_eip = ctxt->_eip;
		ctxt->_eip = ctxt->src.val;
		ctxt->src.val = old_eip;
1776
		rc = em_push(ctxt);
1777 1778
		break;
	}
1779
	case 4: /* jmp abs */
1780
		ctxt->_eip = ctxt->src.val;
1781
		break;
1782 1783 1784
	case 5: /* jmp far */
		rc = em_jmp_far(ctxt);
		break;
1785
	case 6:	/* push */
1786
		rc = em_push(ctxt);
1787 1788
		break;
	}
1789
	return rc;
1790 1791
}

1792
static int em_grp9(struct x86_emulate_ctxt *ctxt)
1793
{
1794
	u64 old = ctxt->dst.orig_val64;
1795

1796 1797 1798 1799
	if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
	    ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
		ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
		ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1800
		ctxt->eflags &= ~EFLG_ZF;
1801
	} else {
1802 1803
		ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
			(u32) ctxt->regs[VCPU_REGS_RBX];
1804

1805
		ctxt->eflags |= EFLG_ZF;
1806
	}
1807
	return X86EMUL_CONTINUE;
1808 1809
}

1810 1811
static int em_ret(struct x86_emulate_ctxt *ctxt)
{
1812 1813 1814
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
1815 1816 1817
	return em_pop(ctxt);
}

1818
static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1819 1820 1821 1822
{
	int rc;
	unsigned long cs;

1823
	rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1824
	if (rc != X86EMUL_CONTINUE)
1825
		return rc;
1826 1827 1828
	if (ctxt->op_bytes == 4)
		ctxt->_eip = (u32)ctxt->_eip;
	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1829
	if (rc != X86EMUL_CONTINUE)
1830
		return rc;
1831
	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1832 1833 1834
	return rc;
}

1835
static int em_lseg(struct x86_emulate_ctxt *ctxt)
1836
{
1837
	int seg = ctxt->src2.val;
1838 1839 1840
	unsigned short sel;
	int rc;

1841
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1842

1843
	rc = load_segment_descriptor(ctxt, sel, seg);
1844 1845 1846
	if (rc != X86EMUL_CONTINUE)
		return rc;

1847
	ctxt->dst.val = ctxt->src.val;
1848 1849 1850
	return rc;
}

1851
static void
1852
setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1853
			struct desc_struct *cs, struct desc_struct *ss)
1854
{
1855 1856
	u16 selector;

1857
	memset(cs, 0, sizeof(struct desc_struct));
1858
	ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1859
	memset(ss, 0, sizeof(struct desc_struct));
1860 1861

	cs->l = 0;		/* will be adjusted later */
1862
	set_desc_base(cs, 0);	/* flat segment */
1863
	cs->g = 1;		/* 4kb granularity */
1864
	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
1865 1866 1867
	cs->type = 0x0b;	/* Read, Execute, Accessed */
	cs->s = 1;
	cs->dpl = 0;		/* will be adjusted later */
1868 1869
	cs->p = 1;
	cs->d = 1;
1870

1871 1872
	set_desc_base(ss, 0);	/* flat segment */
	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
1873 1874 1875
	ss->g = 1;		/* 4kb granularity */
	ss->s = 1;
	ss->type = 0x03;	/* Read/Write, Accessed */
1876
	ss->d = 1;		/* 32bit stack segment */
1877
	ss->dpl = 0;
1878
	ss->p = 1;
1879 1880
}

1881
static int em_syscall(struct x86_emulate_ctxt *ctxt)
1882
{
1883
	struct x86_emulate_ops *ops = ctxt->ops;
1884
	struct desc_struct cs, ss;
1885
	u64 msr_data;
1886
	u16 cs_sel, ss_sel;
1887
	u64 efer = 0;
1888 1889

	/* syscall is not available in real mode */
1890
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1891 1892
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_ud(ctxt);
1893

1894
	ops->get_msr(ctxt, MSR_EFER, &efer);
1895
	setup_syscalls_segments(ctxt, &cs, &ss);
1896

1897
	ops->get_msr(ctxt, MSR_STAR, &msr_data);
1898
	msr_data >>= 32;
1899 1900
	cs_sel = (u16)(msr_data & 0xfffc);
	ss_sel = (u16)(msr_data + 8);
1901

1902
	if (efer & EFER_LMA) {
1903
		cs.d = 0;
1904 1905
		cs.l = 1;
	}
1906 1907
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1908

1909
	ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1910
	if (efer & EFER_LMA) {
1911
#ifdef CONFIG_X86_64
1912
		ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1913

1914
		ops->get_msr(ctxt,
1915 1916
			     ctxt->mode == X86EMUL_MODE_PROT64 ?
			     MSR_LSTAR : MSR_CSTAR, &msr_data);
1917
		ctxt->_eip = msr_data;
1918

1919
		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1920 1921 1922 1923
		ctxt->eflags &= ~(msr_data | EFLG_RF);
#endif
	} else {
		/* legacy mode */
1924
		ops->get_msr(ctxt, MSR_STAR, &msr_data);
1925
		ctxt->_eip = (u32)msr_data;
1926 1927 1928 1929

		ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
	}

1930
	return X86EMUL_CONTINUE;
1931 1932
}

1933
static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1934
{
1935
	struct x86_emulate_ops *ops = ctxt->ops;
1936
	struct desc_struct cs, ss;
1937
	u64 msr_data;
1938
	u16 cs_sel, ss_sel;
1939
	u64 efer = 0;
1940

1941
	ops->get_msr(ctxt, MSR_EFER, &efer);
1942
	/* inject #GP if in real mode */
1943 1944
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return emulate_gp(ctxt, 0);
1945 1946 1947 1948

	/* XXX sysenter/sysexit have not been tested in 64bit mode.
	* Therefore, we inject an #UD.
	*/
1949 1950
	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return emulate_ud(ctxt);
1951

1952
	setup_syscalls_segments(ctxt, &cs, &ss);
1953

1954
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1955 1956
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT32:
1957 1958
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
1959 1960
		break;
	case X86EMUL_MODE_PROT64:
1961 1962
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
1963 1964 1965 1966
		break;
	}

	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1967 1968 1969 1970
	cs_sel = (u16)msr_data;
	cs_sel &= ~SELECTOR_RPL_MASK;
	ss_sel = cs_sel + 8;
	ss_sel &= ~SELECTOR_RPL_MASK;
1971
	if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1972
		cs.d = 0;
1973 1974 1975
		cs.l = 1;
	}

1976 1977
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1978

1979
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1980
	ctxt->_eip = msr_data;
1981

1982
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1983
	ctxt->regs[VCPU_REGS_RSP] = msr_data;
1984

1985
	return X86EMUL_CONTINUE;
1986 1987
}

1988
static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1989
{
1990
	struct x86_emulate_ops *ops = ctxt->ops;
1991
	struct desc_struct cs, ss;
1992 1993
	u64 msr_data;
	int usermode;
1994
	u16 cs_sel = 0, ss_sel = 0;
1995

1996 1997
	/* inject #GP if in real mode or Virtual 8086 mode */
	if (ctxt->mode == X86EMUL_MODE_REAL ||
1998 1999
	    ctxt->mode == X86EMUL_MODE_VM86)
		return emulate_gp(ctxt, 0);
2000

2001
	setup_syscalls_segments(ctxt, &cs, &ss);
2002

2003
	if ((ctxt->rex_prefix & 0x8) != 0x0)
2004 2005 2006 2007 2008 2009
		usermode = X86EMUL_MODE_PROT64;
	else
		usermode = X86EMUL_MODE_PROT32;

	cs.dpl = 3;
	ss.dpl = 3;
2010
	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2011 2012
	switch (usermode) {
	case X86EMUL_MODE_PROT32:
2013
		cs_sel = (u16)(msr_data + 16);
2014 2015
		if ((msr_data & 0xfffc) == 0x0)
			return emulate_gp(ctxt, 0);
2016
		ss_sel = (u16)(msr_data + 24);
2017 2018
		break;
	case X86EMUL_MODE_PROT64:
2019
		cs_sel = (u16)(msr_data + 32);
2020 2021
		if (msr_data == 0x0)
			return emulate_gp(ctxt, 0);
2022 2023
		ss_sel = cs_sel + 8;
		cs.d = 0;
2024 2025 2026
		cs.l = 1;
		break;
	}
2027 2028
	cs_sel |= SELECTOR_RPL_MASK;
	ss_sel |= SELECTOR_RPL_MASK;
2029

2030 2031
	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2032

2033 2034
	ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
	ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2035

2036
	return X86EMUL_CONTINUE;
2037 2038
}

2039
static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2040 2041 2042 2043 2044 2045 2046
{
	int iopl;
	if (ctxt->mode == X86EMUL_MODE_REAL)
		return false;
	if (ctxt->mode == X86EMUL_MODE_VM86)
		return true;
	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2047
	return ctxt->ops->cpl(ctxt) > iopl;
2048 2049 2050 2051 2052
}

static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
					    u16 port, u16 len)
{
2053
	struct x86_emulate_ops *ops = ctxt->ops;
2054
	struct desc_struct tr_seg;
2055
	u32 base3;
2056
	int r;
2057
	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2058
	unsigned mask = (1 << len) - 1;
2059
	unsigned long base;
2060

2061
	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2062
	if (!tr_seg.p)
2063
		return false;
2064
	if (desc_limit_scaled(&tr_seg) < 103)
2065
		return false;
2066 2067 2068 2069
	base = get_desc_base(&tr_seg);
#ifdef CONFIG_X86_64
	base |= ((u64)base3) << 32;
#endif
2070
	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2071 2072
	if (r != X86EMUL_CONTINUE)
		return false;
2073
	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2074
		return false;
2075
	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
	if (r != X86EMUL_CONTINUE)
		return false;
	if ((perm >> bit_idx) & mask)
		return false;
	return true;
}

static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
				 u16 port, u16 len)
{
2086 2087 2088
	if (ctxt->perm_ok)
		return true;

2089 2090
	if (emulator_bad_iopl(ctxt))
		if (!emulator_io_port_access_allowed(ctxt, port, len))
2091
			return false;
2092 2093 2094

	ctxt->perm_ok = true;

2095 2096 2097
	return true;
}

2098 2099 2100
static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_16 *tss)
{
2101
	tss->ip = ctxt->_eip;
2102
	tss->flag = ctxt->eflags;
2103 2104 2105 2106 2107 2108 2109 2110
	tss->ax = ctxt->regs[VCPU_REGS_RAX];
	tss->cx = ctxt->regs[VCPU_REGS_RCX];
	tss->dx = ctxt->regs[VCPU_REGS_RDX];
	tss->bx = ctxt->regs[VCPU_REGS_RBX];
	tss->sp = ctxt->regs[VCPU_REGS_RSP];
	tss->bp = ctxt->regs[VCPU_REGS_RBP];
	tss->si = ctxt->regs[VCPU_REGS_RSI];
	tss->di = ctxt->regs[VCPU_REGS_RDI];
2111

2112 2113 2114 2115 2116
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2117 2118 2119 2120 2121 2122 2123
}

static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_16 *tss)
{
	int ret;

2124
	ctxt->_eip = tss->ip;
2125
	ctxt->eflags = tss->flag | 2;
2126 2127 2128 2129 2130 2131 2132 2133
	ctxt->regs[VCPU_REGS_RAX] = tss->ax;
	ctxt->regs[VCPU_REGS_RCX] = tss->cx;
	ctxt->regs[VCPU_REGS_RDX] = tss->dx;
	ctxt->regs[VCPU_REGS_RBX] = tss->bx;
	ctxt->regs[VCPU_REGS_RSP] = tss->sp;
	ctxt->regs[VCPU_REGS_RBP] = tss->bp;
	ctxt->regs[VCPU_REGS_RSI] = tss->si;
	ctxt->regs[VCPU_REGS_RDI] = tss->di;
2134 2135 2136 2137 2138

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2139 2140 2141 2142 2143
	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2144 2145 2146 2147 2148

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2149
	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2150 2151
	if (ret != X86EMUL_CONTINUE)
		return ret;
2152
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2153 2154
	if (ret != X86EMUL_CONTINUE)
		return ret;
2155
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2156 2157
	if (ret != X86EMUL_CONTINUE)
		return ret;
2158
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2159 2160
	if (ret != X86EMUL_CONTINUE)
		return ret;
2161
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_16(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2172
	struct x86_emulate_ops *ops = ctxt->ops;
2173 2174
	struct tss_segment_16 tss_seg;
	int ret;
2175
	u32 new_tss_base = get_desc_base(new_desc);
2176

2177
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2178
			    &ctxt->exception);
2179
	if (ret != X86EMUL_CONTINUE)
2180 2181 2182
		/* FIXME: need to provide precise fault address */
		return ret;

2183
	save_state_to_tss16(ctxt, &tss_seg);
2184

2185
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2186
			     &ctxt->exception);
2187
	if (ret != X86EMUL_CONTINUE)
2188 2189 2190
		/* FIXME: need to provide precise fault address */
		return ret;

2191
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2192
			    &ctxt->exception);
2193
	if (ret != X86EMUL_CONTINUE)
2194 2195 2196 2197 2198 2199
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2200
		ret = ops->write_std(ctxt, new_tss_base,
2201 2202
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2203
				     &ctxt->exception);
2204
		if (ret != X86EMUL_CONTINUE)
2205 2206 2207 2208
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2209
	return load_state_from_tss16(ctxt, &tss_seg);
2210 2211 2212 2213 2214
}

static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
				struct tss_segment_32 *tss)
{
2215
	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2216
	tss->eip = ctxt->_eip;
2217
	tss->eflags = ctxt->eflags;
2218 2219 2220 2221 2222 2223 2224 2225
	tss->eax = ctxt->regs[VCPU_REGS_RAX];
	tss->ecx = ctxt->regs[VCPU_REGS_RCX];
	tss->edx = ctxt->regs[VCPU_REGS_RDX];
	tss->ebx = ctxt->regs[VCPU_REGS_RBX];
	tss->esp = ctxt->regs[VCPU_REGS_RSP];
	tss->ebp = ctxt->regs[VCPU_REGS_RBP];
	tss->esi = ctxt->regs[VCPU_REGS_RSI];
	tss->edi = ctxt->regs[VCPU_REGS_RDI];
2226

2227 2228 2229 2230 2231 2232 2233
	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
	tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2234 2235 2236 2237 2238 2239 2240
}

static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
				 struct tss_segment_32 *tss)
{
	int ret;

2241
	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2242
		return emulate_gp(ctxt, 0);
2243
	ctxt->_eip = tss->eip;
2244
	ctxt->eflags = tss->eflags | 2;
2245 2246 2247 2248 2249 2250 2251 2252
	ctxt->regs[VCPU_REGS_RAX] = tss->eax;
	ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
	ctxt->regs[VCPU_REGS_RDX] = tss->edx;
	ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
	ctxt->regs[VCPU_REGS_RSP] = tss->esp;
	ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
	ctxt->regs[VCPU_REGS_RSI] = tss->esi;
	ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2253 2254 2255 2256 2257

	/*
	 * SDM says that segment selectors are loaded before segment
	 * descriptors
	 */
2258 2259 2260 2261 2262 2263 2264
	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2265 2266 2267 2268 2269

	/*
	 * Now load segment descriptors. If fault happenes at this stage
	 * it is handled in a context of new task
	 */
2270
	ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2271 2272
	if (ret != X86EMUL_CONTINUE)
		return ret;
2273
	ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2274 2275
	if (ret != X86EMUL_CONTINUE)
		return ret;
2276
	ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2277 2278
	if (ret != X86EMUL_CONTINUE)
		return ret;
2279
	ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2280 2281
	if (ret != X86EMUL_CONTINUE)
		return ret;
2282
	ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2283 2284
	if (ret != X86EMUL_CONTINUE)
		return ret;
2285
	ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2286 2287
	if (ret != X86EMUL_CONTINUE)
		return ret;
2288
	ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298
	if (ret != X86EMUL_CONTINUE)
		return ret;

	return X86EMUL_CONTINUE;
}

static int task_switch_32(struct x86_emulate_ctxt *ctxt,
			  u16 tss_selector, u16 old_tss_sel,
			  ulong old_tss_base, struct desc_struct *new_desc)
{
2299
	struct x86_emulate_ops *ops = ctxt->ops;
2300 2301
	struct tss_segment_32 tss_seg;
	int ret;
2302
	u32 new_tss_base = get_desc_base(new_desc);
2303

2304
	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2305
			    &ctxt->exception);
2306
	if (ret != X86EMUL_CONTINUE)
2307 2308 2309
		/* FIXME: need to provide precise fault address */
		return ret;

2310
	save_state_to_tss32(ctxt, &tss_seg);
2311

2312
	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2313
			     &ctxt->exception);
2314
	if (ret != X86EMUL_CONTINUE)
2315 2316 2317
		/* FIXME: need to provide precise fault address */
		return ret;

2318
	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2319
			    &ctxt->exception);
2320
	if (ret != X86EMUL_CONTINUE)
2321 2322 2323 2324 2325 2326
		/* FIXME: need to provide precise fault address */
		return ret;

	if (old_tss_sel != 0xffff) {
		tss_seg.prev_task_link = old_tss_sel;

2327
		ret = ops->write_std(ctxt, new_tss_base,
2328 2329
				     &tss_seg.prev_task_link,
				     sizeof tss_seg.prev_task_link,
2330
				     &ctxt->exception);
2331
		if (ret != X86EMUL_CONTINUE)
2332 2333 2334 2335
			/* FIXME: need to provide precise fault address */
			return ret;
	}

2336
	return load_state_from_tss32(ctxt, &tss_seg);
2337 2338 2339
}

static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2340 2341
				   u16 tss_selector, int reason,
				   bool has_error_code, u32 error_code)
2342
{
2343
	struct x86_emulate_ops *ops = ctxt->ops;
2344 2345
	struct desc_struct curr_tss_desc, next_tss_desc;
	int ret;
2346
	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2347
	ulong old_tss_base =
2348
		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2349
	u32 desc_limit;
2350 2351 2352

	/* FIXME: old_tss_base == ~0 ? */

2353
	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2354 2355
	if (ret != X86EMUL_CONTINUE)
		return ret;
2356
	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2357 2358 2359 2360 2361 2362 2363
	if (ret != X86EMUL_CONTINUE)
		return ret;

	/* FIXME: check that next_tss_desc is tss */

	if (reason != TASK_SWITCH_IRET) {
		if ((tss_selector & 3) > next_tss_desc.dpl ||
2364
		    ops->cpl(ctxt) > next_tss_desc.dpl)
2365
			return emulate_gp(ctxt, 0);
2366 2367
	}

2368 2369 2370 2371
	desc_limit = desc_limit_scaled(&next_tss_desc);
	if (!next_tss_desc.p ||
	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
	     desc_limit < 0x2b)) {
2372
		emulate_ts(ctxt, tss_selector & 0xfffc);
2373 2374 2375 2376 2377
		return X86EMUL_PROPAGATE_FAULT;
	}

	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2378
		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389
	}

	if (reason == TASK_SWITCH_IRET)
		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;

	/* set back link to prev task only if NT bit is set in eflags
	   note that old_tss_sel is not used afetr this point */
	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
		old_tss_sel = 0xffff;

	if (next_tss_desc.type & 8)
2390
		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2391 2392
				     old_tss_base, &next_tss_desc);
	else
2393
		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2394
				     old_tss_base, &next_tss_desc);
2395 2396
	if (ret != X86EMUL_CONTINUE)
		return ret;
2397 2398 2399 2400 2401 2402

	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;

	if (reason != TASK_SWITCH_IRET) {
		next_tss_desc.type |= (1 << 1); /* set busy flag */
2403
		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2404 2405
	}

2406
	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
2407
	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2408

2409
	if (has_error_code) {
2410 2411 2412
		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
		ctxt->lock_prefix = 0;
		ctxt->src.val = (unsigned long) error_code;
2413
		ret = em_push(ctxt);
2414 2415
	}

2416 2417 2418 2419
	return ret;
}

int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2420 2421
			 u16 tss_selector, int reason,
			 bool has_error_code, u32 error_code)
2422 2423 2424
{
	int rc;

2425 2426
	ctxt->_eip = ctxt->eip;
	ctxt->dst.type = OP_NONE;
2427

2428
	rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2429
				     has_error_code, error_code);
2430

2431
	if (rc == X86EMUL_CONTINUE)
2432
		ctxt->eip = ctxt->_eip;
2433

2434
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2435 2436
}

2437
static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2438
			    int reg, struct operand *op)
2439 2440 2441
{
	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;

2442 2443
	register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
	op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2444
	op->addr.mem.seg = seg;
2445 2446
}

2447 2448 2449 2450 2451 2452
static int em_das(struct x86_emulate_ctxt *ctxt)
{
	u8 al, old_al;
	bool af, cf, old_cf;

	cf = ctxt->eflags & X86_EFLAGS_CF;
2453
	al = ctxt->dst.val;
2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470

	old_al = al;
	old_cf = cf;
	cf = false;
	af = ctxt->eflags & X86_EFLAGS_AF;
	if ((al & 0x0f) > 9 || af) {
		al -= 6;
		cf = old_cf | (al >= 250);
		af = true;
	} else {
		af = false;
	}
	if (old_al > 0x99 || old_cf) {
		al -= 0x60;
		cf = true;
	}

2471
	ctxt->dst.val = al;
2472
	/* Set PF, ZF, SF */
2473 2474 2475
	ctxt->src.type = OP_IMM;
	ctxt->src.val = 0;
	ctxt->src.bytes = 1;
2476
	emulate_2op_SrcV(ctxt, "or");
2477 2478 2479 2480 2481 2482 2483 2484
	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
	if (cf)
		ctxt->eflags |= X86_EFLAGS_CF;
	if (af)
		ctxt->eflags |= X86_EFLAGS_AF;
	return X86EMUL_CONTINUE;
}

2485 2486 2487 2488 2489 2490
static int em_call_far(struct x86_emulate_ctxt *ctxt)
{
	u16 sel, old_cs;
	ulong old_eip;
	int rc;

2491
	old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2492
	old_eip = ctxt->_eip;
2493

2494
	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2495
	if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2496 2497
		return X86EMUL_CONTINUE;

2498 2499
	ctxt->_eip = 0;
	memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2500

2501
	ctxt->src.val = old_cs;
2502
	rc = em_push(ctxt);
2503 2504 2505
	if (rc != X86EMUL_CONTINUE)
		return rc;

2506
	ctxt->src.val = old_eip;
2507
	return em_push(ctxt);
2508 2509
}

2510 2511 2512 2513
static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2514 2515 2516 2517
	ctxt->dst.type = OP_REG;
	ctxt->dst.addr.reg = &ctxt->_eip;
	ctxt->dst.bytes = ctxt->op_bytes;
	rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2518 2519
	if (rc != X86EMUL_CONTINUE)
		return rc;
2520
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2521 2522 2523
	return X86EMUL_CONTINUE;
}

2524 2525
static int em_add(struct x86_emulate_ctxt *ctxt)
{
2526
	emulate_2op_SrcV(ctxt, "add");
2527 2528 2529 2530 2531
	return X86EMUL_CONTINUE;
}

static int em_or(struct x86_emulate_ctxt *ctxt)
{
2532
	emulate_2op_SrcV(ctxt, "or");
2533 2534 2535 2536 2537
	return X86EMUL_CONTINUE;
}

static int em_adc(struct x86_emulate_ctxt *ctxt)
{
2538
	emulate_2op_SrcV(ctxt, "adc");
2539 2540 2541 2542 2543
	return X86EMUL_CONTINUE;
}

static int em_sbb(struct x86_emulate_ctxt *ctxt)
{
2544
	emulate_2op_SrcV(ctxt, "sbb");
2545 2546 2547 2548 2549
	return X86EMUL_CONTINUE;
}

static int em_and(struct x86_emulate_ctxt *ctxt)
{
2550
	emulate_2op_SrcV(ctxt, "and");
2551 2552 2553 2554 2555
	return X86EMUL_CONTINUE;
}

static int em_sub(struct x86_emulate_ctxt *ctxt)
{
2556
	emulate_2op_SrcV(ctxt, "sub");
2557 2558 2559 2560 2561
	return X86EMUL_CONTINUE;
}

static int em_xor(struct x86_emulate_ctxt *ctxt)
{
2562
	emulate_2op_SrcV(ctxt, "xor");
2563 2564 2565 2566 2567
	return X86EMUL_CONTINUE;
}

static int em_cmp(struct x86_emulate_ctxt *ctxt)
{
2568
	emulate_2op_SrcV(ctxt, "cmp");
2569
	/* Disable writeback. */
2570
	ctxt->dst.type = OP_NONE;
2571 2572 2573
	return X86EMUL_CONTINUE;
}

2574 2575
static int em_test(struct x86_emulate_ctxt *ctxt)
{
2576
	emulate_2op_SrcV(ctxt, "test");
2577 2578
	/* Disable writeback. */
	ctxt->dst.type = OP_NONE;
2579 2580 2581
	return X86EMUL_CONTINUE;
}

2582 2583 2584
static int em_xchg(struct x86_emulate_ctxt *ctxt)
{
	/* Write back the register source. */
2585 2586
	ctxt->src.val = ctxt->dst.val;
	write_register_operand(&ctxt->src);
2587 2588

	/* Write back the memory destination with implicit LOCK prefix. */
2589 2590
	ctxt->dst.val = ctxt->src.orig_val;
	ctxt->lock_prefix = 1;
2591 2592 2593
	return X86EMUL_CONTINUE;
}

2594
static int em_imul(struct x86_emulate_ctxt *ctxt)
2595
{
2596
	emulate_2op_SrcV_nobyte(ctxt, "imul");
2597 2598 2599
	return X86EMUL_CONTINUE;
}

2600 2601
static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
{
2602
	ctxt->dst.val = ctxt->src2.val;
2603 2604 2605
	return em_imul(ctxt);
}

2606 2607
static int em_cwd(struct x86_emulate_ctxt *ctxt)
{
2608 2609 2610 2611
	ctxt->dst.type = OP_REG;
	ctxt->dst.bytes = ctxt->src.bytes;
	ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2612 2613 2614 2615

	return X86EMUL_CONTINUE;
}

2616 2617 2618 2619
static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
{
	u64 tsc = 0;

2620
	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2621 2622
	ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
	ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2623 2624 2625
	return X86EMUL_CONTINUE;
}

2626 2627
static int em_mov(struct x86_emulate_ctxt *ctxt)
{
2628
	ctxt->dst.val = ctxt->src.val;
2629 2630 2631
	return X86EMUL_CONTINUE;
}

2632 2633
static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
{
2634
	if (ctxt->modrm_reg > VCPU_SREG_GS)
2635 2636
		return emulate_ud(ctxt);

2637
	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2638 2639 2640 2641 2642
	return X86EMUL_CONTINUE;
}

static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
{
2643
	u16 sel = ctxt->src.val;
2644

2645
	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2646 2647
		return emulate_ud(ctxt);

2648
	if (ctxt->modrm_reg == VCPU_SREG_SS)
2649 2650 2651
		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;

	/* Disable writeback. */
2652 2653
	ctxt->dst.type = OP_NONE;
	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2654 2655
}

2656 2657
static int em_movdqu(struct x86_emulate_ctxt *ctxt)
{
2658
	memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2659 2660 2661
	return X86EMUL_CONTINUE;
}

2662 2663
static int em_invlpg(struct x86_emulate_ctxt *ctxt)
{
2664 2665 2666
	int rc;
	ulong linear;

2667
	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2668
	if (rc == X86EMUL_CONTINUE)
2669
		ctxt->ops->invlpg(ctxt, linear);
2670
	/* Disable writeback. */
2671
	ctxt->dst.type = OP_NONE;
2672 2673 2674
	return X86EMUL_CONTINUE;
}

2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
static int em_clts(struct x86_emulate_ctxt *ctxt)
{
	ulong cr0;

	cr0 = ctxt->ops->get_cr(ctxt, 0);
	cr0 &= ~X86_CR0_TS;
	ctxt->ops->set_cr(ctxt, 0, cr0);
	return X86EMUL_CONTINUE;
}

2685 2686 2687 2688
static int em_vmcall(struct x86_emulate_ctxt *ctxt)
{
	int rc;

2689
	if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2690 2691 2692 2693 2694 2695 2696
		return X86EMUL_UNHANDLEABLE;

	rc = ctxt->ops->fix_hypercall(ctxt);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	/* Let the processor re-execute the fixed hypercall */
2697
	ctxt->_eip = ctxt->eip;
2698
	/* Disable writeback. */
2699
	ctxt->dst.type = OP_NONE;
2700 2701 2702 2703 2704 2705 2706 2707
	return X86EMUL_CONTINUE;
}

static int em_lgdt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2708
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2709
			     &desc_ptr.size, &desc_ptr.address,
2710
			     ctxt->op_bytes);
2711 2712 2713 2714
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_gdt(ctxt, &desc_ptr);
	/* Disable writeback. */
2715
	ctxt->dst.type = OP_NONE;
2716 2717 2718
	return X86EMUL_CONTINUE;
}

2719
static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2720 2721 2722
{
	int rc;

2723 2724
	rc = ctxt->ops->fix_hypercall(ctxt);

2725
	/* Disable writeback. */
2726
	ctxt->dst.type = OP_NONE;
2727 2728 2729 2730 2731 2732 2733 2734
	return rc;
}

static int em_lidt(struct x86_emulate_ctxt *ctxt)
{
	struct desc_ptr desc_ptr;
	int rc;

2735
	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2736
			     &desc_ptr.size, &desc_ptr.address,
2737
			     ctxt->op_bytes);
2738 2739 2740 2741
	if (rc != X86EMUL_CONTINUE)
		return rc;
	ctxt->ops->set_idt(ctxt, &desc_ptr);
	/* Disable writeback. */
2742
	ctxt->dst.type = OP_NONE;
2743 2744 2745 2746 2747
	return X86EMUL_CONTINUE;
}

static int em_smsw(struct x86_emulate_ctxt *ctxt)
{
2748 2749
	ctxt->dst.bytes = 2;
	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2750 2751 2752 2753 2754 2755
	return X86EMUL_CONTINUE;
}

static int em_lmsw(struct x86_emulate_ctxt *ctxt)
{
	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2756 2757
			  | (ctxt->src.val & 0x0f));
	ctxt->dst.type = OP_NONE;
2758 2759 2760
	return X86EMUL_CONTINUE;
}

2761 2762
static int em_loop(struct x86_emulate_ctxt *ctxt)
{
2763 2764 2765 2766
	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
	if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
		jmp_rel(ctxt, ctxt->src.val);
2767 2768 2769 2770 2771 2772

	return X86EMUL_CONTINUE;
}

static int em_jcxz(struct x86_emulate_ctxt *ctxt)
{
2773 2774
	if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
		jmp_rel(ctxt, ctxt->src.val);
2775 2776 2777 2778

	return X86EMUL_CONTINUE;
}

2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797
static int em_cli(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->eflags &= ~X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

static int em_sti(struct x86_emulate_ctxt *ctxt)
{
	if (emulator_bad_iopl(ctxt))
		return emulate_gp(ctxt, 0);

	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
	ctxt->eflags |= X86_EFLAGS_IF;
	return X86EMUL_CONTINUE;
}

2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
static bool valid_cr(int nr)
{
	switch (nr) {
	case 0:
	case 2 ... 4:
	case 8:
		return true;
	default:
		return false;
	}
}

static int check_cr_read(struct x86_emulate_ctxt *ctxt)
{
2812
	if (!valid_cr(ctxt->modrm_reg))
2813 2814 2815 2816 2817 2818 2819
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_cr_write(struct x86_emulate_ctxt *ctxt)
{
2820 2821
	u64 new_val = ctxt->src.val64;
	int cr = ctxt->modrm_reg;
2822
	u64 efer = 0;
2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839

	static u64 cr_reserved_bits[] = {
		0xffffffff00000000ULL,
		0, 0, 0, /* CR3 checked later */
		CR4_RESERVED_BITS,
		0, 0, 0,
		CR8_RESERVED_BITS,
	};

	if (!valid_cr(cr))
		return emulate_ud(ctxt);

	if (new_val & cr_reserved_bits[cr])
		return emulate_gp(ctxt, 0);

	switch (cr) {
	case 0: {
2840
		u64 cr4;
2841 2842 2843 2844
		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
			return emulate_gp(ctxt, 0);

2845 2846
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2847 2848 2849 2850 2851 2852 2853 2854 2855 2856

		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
		    !(cr4 & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	case 3: {
		u64 rsvd = 0;

2857 2858
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
		if (efer & EFER_LMA)
2859
			rsvd = CR3_L_MODE_RESERVED_BITS;
2860
		else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2861
			rsvd = CR3_PAE_RESERVED_BITS;
2862
		else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2863 2864 2865 2866 2867 2868 2869 2870
			rsvd = CR3_NONPAE_RESERVED_BITS;

		if (new_val & rsvd)
			return emulate_gp(ctxt, 0);

		break;
		}
	case 4: {
2871
		u64 cr4;
2872

2873 2874
		cr4 = ctxt->ops->get_cr(ctxt, 4);
		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885

		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
			return emulate_gp(ctxt, 0);

		break;
		}
	}

	return X86EMUL_CONTINUE;
}

2886 2887 2888 2889
static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
{
	unsigned long dr7;

2890
	ctxt->ops->get_dr(ctxt, 7, &dr7);
2891 2892 2893 2894 2895 2896 2897

	/* Check if DR7.Global_Enable is set */
	return dr7 & (1 << 13);
}

static int check_dr_read(struct x86_emulate_ctxt *ctxt)
{
2898
	int dr = ctxt->modrm_reg;
2899 2900 2901 2902 2903
	u64 cr4;

	if (dr > 7)
		return emulate_ud(ctxt);

2904
	cr4 = ctxt->ops->get_cr(ctxt, 4);
2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915
	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
		return emulate_ud(ctxt);

	if (check_dr7_gd(ctxt))
		return emulate_db(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_dr_write(struct x86_emulate_ctxt *ctxt)
{
2916 2917
	u64 new_val = ctxt->src.val64;
	int dr = ctxt->modrm_reg;
2918 2919 2920 2921 2922 2923 2924

	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
		return emulate_gp(ctxt, 0);

	return check_dr_read(ctxt);
}

2925 2926 2927 2928
static int check_svme(struct x86_emulate_ctxt *ctxt)
{
	u64 efer;

2929
	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2930 2931 2932 2933 2934 2935 2936 2937 2938

	if (!(efer & EFER_SVME))
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
{
2939
	u64 rax = ctxt->regs[VCPU_REGS_RAX];
2940 2941

	/* Valid physical address? */
2942
	if (rax & 0xffff000000000000ULL)
2943 2944 2945 2946 2947
		return emulate_gp(ctxt, 0);

	return check_svme(ctxt);
}

2948 2949
static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
{
2950
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2951

2952
	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2953 2954 2955 2956 2957
		return emulate_ud(ctxt);

	return X86EMUL_CONTINUE;
}

2958 2959
static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
{
2960
	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2961
	u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2962

2963
	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2964 2965 2966 2967 2968 2969
	    (rcx > 3))
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2970 2971
static int check_perm_in(struct x86_emulate_ctxt *ctxt)
{
2972 2973
	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2974 2975 2976 2977 2978 2979 2980
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

static int check_perm_out(struct x86_emulate_ctxt *ctxt)
{
2981 2982
	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2983 2984 2985 2986 2987
		return emulate_gp(ctxt, 0);

	return X86EMUL_CONTINUE;
}

2988
#define D(_y) { .flags = (_y) }
2989
#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2990 2991
#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
		      .check_perm = (_p) }
2992
#define N    D(0)
2993
#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2994
#define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2995
#define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2996
#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2997 2998
#define II(_f, _e, _i) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2999 3000 3001
#define IIP(_f, _e, _i, _p) \
	{ .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
	  .check_perm = (_p) }
3002
#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3003

3004
#define D2bv(_f)      D((_f) | ByteOp), D(_f)
3005
#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3006 3007
#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)

3008 3009 3010
#define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
		I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
		I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3011

3012 3013 3014 3015 3016 3017
static struct opcode group7_rm1[] = {
	DI(SrcNone | ModRM | Priv, monitor),
	DI(SrcNone | ModRM | Priv, mwait),
	N, N, N, N, N, N,
};

3018 3019
static struct opcode group7_rm3[] = {
	DIP(SrcNone | ModRM | Prot | Priv, vmrun,   check_svme_pa),
3020
	II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3021 3022 3023 3024 3025 3026 3027
	DIP(SrcNone | ModRM | Prot | Priv, vmload,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, vmsave,  check_svme_pa),
	DIP(SrcNone | ModRM | Prot | Priv, stgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, clgi,    check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, skinit,  check_svme),
	DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
};
3028

3029 3030 3031 3032 3033
static struct opcode group7_rm7[] = {
	N,
	DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
	N, N, N, N, N, N,
};
3034

3035
static struct opcode group1[] = {
3036
	I(Lock, em_add),
3037
	I(Lock | PageTable, em_or),
3038 3039
	I(Lock, em_adc),
	I(Lock, em_sbb),
3040
	I(Lock | PageTable, em_and),
3041 3042 3043
	I(Lock, em_sub),
	I(Lock, em_xor),
	I(0, em_cmp),
3044 3045 3046 3047 3048 3049 3050
};

static struct opcode group1A[] = {
	D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
};

static struct opcode group3[] = {
3051 3052 3053 3054 3055 3056 3057 3058
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcImm | ModRM, em_test),
	I(DstMem | SrcNone | ModRM | Lock, em_not),
	I(DstMem | SrcNone | ModRM | Lock, em_neg),
	I(SrcMem | ModRM, em_mul_ex),
	I(SrcMem | ModRM, em_imul_ex),
	I(SrcMem | ModRM, em_div_ex),
	I(SrcMem | ModRM, em_idiv_ex),
3059 3060 3061 3062 3063 3064 3065 3066 3067
};

static struct opcode group4[] = {
	D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
	N, N, N, N, N, N,
};

static struct opcode group5[] = {
	D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3068 3069
	D(SrcMem | ModRM | Stack),
	I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3070 3071 3072 3073
	D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
	D(SrcMem | ModRM | Stack), N,
};

3074 3075 3076 3077 3078 3079 3080 3081
static struct opcode group6[] = {
	DI(ModRM | Prot,        sldt),
	DI(ModRM | Prot,        str),
	DI(ModRM | Prot | Priv, lldt),
	DI(ModRM | Prot | Priv, ltr),
	N, N, N, N,
};

3082
static struct group_dual group7 = { {
3083 3084
	DI(ModRM | Mov | DstMem | Priv, sgdt),
	DI(ModRM | Mov | DstMem | Priv, sidt),
3085 3086 3087 3088 3089
	II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
	II(ModRM | SrcMem | Priv, em_lidt, lidt),
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
	II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3090
}, {
3091 3092
	I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
	EXT(0, group7_rm1),
3093
	N, EXT(0, group7_rm3),
3094 3095
	II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
	II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3096 3097 3098 3099
} };

static struct opcode group8[] = {
	N, N, N, N,
3100 3101 3102 3103
	D(DstMem | SrcImmByte | ModRM),
	D(DstMem | SrcImmByte | ModRM | Lock | PageTable),
	D(DstMem | SrcImmByte | ModRM | Lock),
	D(DstMem | SrcImmByte | ModRM | Lock | PageTable),
3104 3105 3106
};

static struct group_dual group9 = { {
3107
	N, D(DstMem64 | ModRM | Lock | PageTable), N, N, N, N, N, N,
3108 3109 3110 3111
}, {
	N, N, N, N, N, N, N, N,
} };

3112
static struct opcode group11[] = {
3113 3114
	I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
	X7(D(Undefined)),
3115 3116
};

3117 3118 3119 3120
static struct gprefix pfx_0f_6f_0f_7f = {
	N, N, N, I(Sse, em_movdqu),
};

3121 3122
static struct opcode opcode_table[256] = {
	/* 0x00 - 0x07 */
3123
	I6ALU(Lock, em_add),
3124 3125
	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3126
	/* 0x08 - 0x0F */
3127
	I6ALU(Lock | PageTable, em_or),
3128 3129
	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
	N,
3130
	/* 0x10 - 0x17 */
3131
	I6ALU(Lock, em_adc),
3132 3133
	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3134
	/* 0x18 - 0x1F */
3135
	I6ALU(Lock, em_sbb),
3136 3137
	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3138
	/* 0x20 - 0x27 */
3139
	I6ALU(Lock | PageTable, em_and), N, N,
3140
	/* 0x28 - 0x2F */
3141
	I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3142
	/* 0x30 - 0x37 */
3143
	I6ALU(Lock, em_xor), N, N,
3144
	/* 0x38 - 0x3F */
3145
	I6ALU(0, em_cmp), N, N,
3146 3147 3148
	/* 0x40 - 0x4F */
	X16(D(DstReg)),
	/* 0x50 - 0x57 */
3149
	X8(I(SrcReg | Stack, em_push)),
3150
	/* 0x58 - 0x5F */
3151
	X8(I(DstReg | Stack, em_pop)),
3152
	/* 0x60 - 0x67 */
3153 3154
	I(ImplicitOps | Stack | No64, em_pusha),
	I(ImplicitOps | Stack | No64, em_popa),
3155 3156 3157
	N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
	N, N, N, N,
	/* 0x68 - 0x6F */
3158 3159
	I(SrcImm | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3160 3161
	I(SrcImmByte | Mov | Stack, em_push),
	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3162 3163
	D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
	D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3164 3165 3166 3167 3168 3169 3170
	/* 0x70 - 0x7F */
	X16(D(SrcImmByte)),
	/* 0x80 - 0x87 */
	G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
	G(DstMem | SrcImm | ModRM | Group, group1),
	G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
	G(DstMem | SrcImmByte | ModRM | Group, group1),
3171
	I2bv(DstMem | SrcReg | ModRM, em_test),
3172
	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3173
	/* 0x88 - 0x8F */
3174
	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3175
	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3176
	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3177 3178 3179
	D(ModRM | SrcMem | NoAccess | DstReg),
	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
	G(0, group1A),
3180
	/* 0x90 - 0x97 */
3181
	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3182
	/* 0x98 - 0x9F */
3183
	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3184
	I(SrcImmFAddr | No64, em_call_far), N,
3185 3186
	II(ImplicitOps | Stack, em_pushf, pushf),
	II(ImplicitOps | Stack, em_popf, popf), N, N,
3187
	/* 0xA0 - 0xA7 */
3188
	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3189
	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3190
	I2bv(SrcSI | DstDI | Mov | String, em_mov),
3191
	I2bv(SrcSI | DstDI | String, em_cmp),
3192
	/* 0xA8 - 0xAF */
3193
	I2bv(DstAcc | SrcImm, em_test),
3194 3195
	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3196
	I2bv(SrcAcc | DstDI | String, em_cmp),
3197
	/* 0xB0 - 0xB7 */
3198
	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3199
	/* 0xB8 - 0xBF */
3200
	X8(I(DstReg | SrcImm | Mov, em_mov)),
3201
	/* 0xC0 - 0xC7 */
3202
	D2bv(DstMem | SrcImmByte | ModRM),
3203
	I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3204
	I(ImplicitOps | Stack, em_ret),
3205 3206
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3207
	G(ByteOp, group11), G(0, group11),
3208
	/* 0xC8 - 0xCF */
3209
	N, N, N, I(ImplicitOps | Stack, em_ret_far),
3210
	D(ImplicitOps), DI(SrcImmByte, intn),
3211
	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3212
	/* 0xD0 - 0xD7 */
3213
	D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3214 3215 3216 3217
	N, N, N, N,
	/* 0xD8 - 0xDF */
	N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xE7 */
3218 3219
	X3(I(SrcImmByte, em_loop)),
	I(SrcImmByte, em_jcxz),
3220 3221
	D2bvIP(SrcImmUByte | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3222 3223
	/* 0xE8 - 0xEF */
	D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3224
	I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3225 3226
	D2bvIP(SrcDX | DstAcc, in,  check_perm_in),
	D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3227
	/* 0xF0 - 0xF7 */
3228
	N, DI(ImplicitOps, icebp), N, N,
3229 3230
	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
	G(ByteOp, group3), G(0, group3),
3231
	/* 0xF8 - 0xFF */
3232 3233
	D(ImplicitOps), D(ImplicitOps),
	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3234 3235 3236 3237 3238
	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
};

static struct opcode twobyte_table[256] = {
	/* 0x00 - 0x0F */
3239
	G(0, group6), GD(0, &group7), N, N,
3240 3241
	N, I(ImplicitOps | VendorSpecific, em_syscall),
	II(ImplicitOps | Priv, em_clts, clts), N,
3242
	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3243 3244 3245 3246
	N, D(ImplicitOps | ModRM), N, N,
	/* 0x10 - 0x1F */
	N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
	/* 0x20 - 0x2F */
3247
	DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3248
	DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3249
	DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3250
	DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3251 3252 3253
	N, N, N, N,
	N, N, N, N, N, N, N, N,
	/* 0x30 - 0x3F */
3254 3255 3256 3257
	DI(ImplicitOps | Priv, wrmsr),
	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
	DI(ImplicitOps | Priv, rdmsr),
	DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3258 3259
	I(ImplicitOps | VendorSpecific, em_sysenter),
	I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3260
	N, N,
3261 3262 3263 3264 3265 3266
	N, N, N, N, N, N, N, N,
	/* 0x40 - 0x4F */
	X16(D(DstReg | SrcMem | ModRM | Mov)),
	/* 0x50 - 0x5F */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0x60 - 0x6F */
3267 3268 3269 3270
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3271
	/* 0x70 - 0x7F */
3272 3273 3274 3275
	N, N, N, N,
	N, N, N, N,
	N, N, N, N,
	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3276 3277 3278
	/* 0x80 - 0x8F */
	X16(D(SrcImm)),
	/* 0x90 - 0x9F */
3279
	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3280
	/* 0xA0 - 0xA7 */
3281
	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3282
	DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3283 3284 3285
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM), N, N,
	/* 0xA8 - 0xAF */
3286
	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3287 3288
	DI(ImplicitOps, rsm),
	D(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable),
3289 3290
	D(DstMem | SrcReg | Src2ImmByte | ModRM),
	D(DstMem | SrcReg | Src2CL | ModRM),
3291
	D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3292
	/* 0xB0 - 0xB7 */
3293
	D2bv(DstMem | SrcReg | ModRM | Lock | PageTable),
3294
	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3295
	D(DstMem | SrcReg | ModRM | BitOp | Lock),
3296 3297
	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3298
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3299 3300
	/* 0xB8 - 0xBF */
	N, N,
3301
	G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable),
3302 3303
	D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
	D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3304
	/* 0xC0 - 0xCF */
3305
	D2bv(DstMem | SrcReg | ModRM | Lock),
3306
	N, D(DstMem | SrcReg | ModRM | Mov),
3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321
	N, N, N, GD(0, &group9),
	N, N, N, N, N, N, N, N,
	/* 0xD0 - 0xDF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xE0 - 0xEF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
	/* 0xF0 - 0xFF */
	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
};

#undef D
#undef N
#undef G
#undef GD
#undef I
3322
#undef GP
3323
#undef EXT
3324

3325
#undef D2bv
3326
#undef D2bvIP
3327
#undef I2bv
3328
#undef I6ALU
3329

3330
static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3331 3332 3333
{
	unsigned size;

3334
	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346
	if (size == 8)
		size = 4;
	return size;
}

static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
		      unsigned size, bool sign_extension)
{
	int rc = X86EMUL_CONTINUE;

	op->type = OP_IMM;
	op->bytes = size;
3347
	op->addr.mem.ea = ctxt->_eip;
3348 3349 3350
	/* NB. Immediates are sign-extended as necessary. */
	switch (op->bytes) {
	case 1:
3351
		op->val = insn_fetch(s8, ctxt);
3352 3353
		break;
	case 2:
3354
		op->val = insn_fetch(s16, ctxt);
3355 3356
		break;
	case 4:
3357
		op->val = insn_fetch(s32, ctxt);
3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376
		break;
	}
	if (!sign_extension) {
		switch (op->bytes) {
		case 1:
			op->val &= 0xff;
			break;
		case 2:
			op->val &= 0xffff;
			break;
		case 4:
			op->val &= 0xffffffff;
			break;
		}
	}
done:
	return rc;
}

3377 3378 3379 3380 3381 3382 3383 3384
static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
			  unsigned d)
{
	int rc = X86EMUL_CONTINUE;

	switch (d) {
	case OpReg:
		decode_register_operand(ctxt, op,
3385
			 op == &ctxt->dst &&
3386 3387 3388
			 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
		break;
	case OpImmUByte:
3389
		rc = decode_imm(ctxt, op, 1, false);
3390 3391
		break;
	case OpMem:
3392
		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3393 3394 3395 3396
	mem_common:
		*op = ctxt->memop;
		ctxt->memopp = op;
		if ((ctxt->d & BitOp) && op == &ctxt->dst)
3397 3398 3399
			fetch_bit_operand(ctxt);
		op->orig_val = op->val;
		break;
3400 3401 3402
	case OpMem64:
		ctxt->memop.bytes = 8;
		goto mem_common;
3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423
	case OpAcc:
		op->type = OP_REG;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
		fetch_register_operand(op);
		op->orig_val = op->val;
		break;
	case OpDI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
		op->addr.mem.seg = VCPU_SREG_ES;
		op->val = 0;
		break;
	case OpDX:
		op->type = OP_REG;
		op->bytes = 2;
		op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
		fetch_register_operand(op);
		break;
3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437
	case OpCL:
		op->bytes = 1;
		op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
		break;
	case OpImmByte:
		rc = decode_imm(ctxt, op, 1, true);
		break;
	case OpOne:
		op->bytes = 1;
		op->val = 1;
		break;
	case OpImm:
		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
		break;
3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466
	case OpMem16:
		ctxt->memop.bytes = 2;
		goto mem_common;
	case OpMem32:
		ctxt->memop.bytes = 4;
		goto mem_common;
	case OpImmU16:
		rc = decode_imm(ctxt, op, 2, false);
		break;
	case OpImmU:
		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
		break;
	case OpSI:
		op->type = OP_MEM;
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
		op->addr.mem.ea =
			register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
		op->addr.mem.seg = seg_override(ctxt);
		op->val = 0;
		break;
	case OpImmFAddr:
		op->type = OP_IMM;
		op->addr.mem.ea = ctxt->_eip;
		op->bytes = ctxt->op_bytes + 2;
		insn_fetch_arr(op->valptr, op->bytes, ctxt);
		break;
	case OpMemFAddr:
		ctxt->memop.bytes = ctxt->op_bytes + 2;
		goto mem_common;
3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
	case OpES:
		op->val = VCPU_SREG_ES;
		break;
	case OpCS:
		op->val = VCPU_SREG_CS;
		break;
	case OpSS:
		op->val = VCPU_SREG_SS;
		break;
	case OpDS:
		op->val = VCPU_SREG_DS;
		break;
	case OpFS:
		op->val = VCPU_SREG_FS;
		break;
	case OpGS:
		op->val = VCPU_SREG_GS;
		break;
3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495
	case OpImplicit:
		/* Special instructions do their own operand decoding. */
	default:
		op->type = OP_NONE; /* Disable writeback. */
		break;
	}

done:
	return rc;
}

3496
int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3497 3498 3499
{
	int rc = X86EMUL_CONTINUE;
	int mode = ctxt->mode;
3500
	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3501
	bool op_prefix = false;
3502
	struct opcode opcode;
3503

3504 3505
	ctxt->memop.type = OP_NONE;
	ctxt->memopp = NULL;
3506 3507 3508
	ctxt->_eip = ctxt->eip;
	ctxt->fetch.start = ctxt->_eip;
	ctxt->fetch.end = ctxt->fetch.start + insn_len;
3509
	if (insn_len > 0)
3510
		memcpy(ctxt->fetch.data, insn, insn_len);
3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527

	switch (mode) {
	case X86EMUL_MODE_REAL:
	case X86EMUL_MODE_VM86:
	case X86EMUL_MODE_PROT16:
		def_op_bytes = def_ad_bytes = 2;
		break;
	case X86EMUL_MODE_PROT32:
		def_op_bytes = def_ad_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86EMUL_MODE_PROT64:
		def_op_bytes = 4;
		def_ad_bytes = 8;
		break;
#endif
	default:
3528
		return EMULATION_FAILED;
3529 3530
	}

3531 3532
	ctxt->op_bytes = def_op_bytes;
	ctxt->ad_bytes = def_ad_bytes;
3533 3534 3535

	/* Legacy prefixes. */
	for (;;) {
3536
		switch (ctxt->b = insn_fetch(u8, ctxt)) {
3537
		case 0x66:	/* operand-size override */
3538
			op_prefix = true;
3539
			/* switch between 2/4 bytes */
3540
			ctxt->op_bytes = def_op_bytes ^ 6;
3541 3542 3543 3544
			break;
		case 0x67:	/* address-size override */
			if (mode == X86EMUL_MODE_PROT64)
				/* switch between 4/8 bytes */
3545
				ctxt->ad_bytes = def_ad_bytes ^ 12;
3546 3547
			else
				/* switch between 2/4 bytes */
3548
				ctxt->ad_bytes = def_ad_bytes ^ 6;
3549 3550 3551 3552 3553
			break;
		case 0x26:	/* ES override */
		case 0x2e:	/* CS override */
		case 0x36:	/* SS override */
		case 0x3e:	/* DS override */
3554
			set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3555 3556 3557
			break;
		case 0x64:	/* FS override */
		case 0x65:	/* GS override */
3558
			set_seg_override(ctxt, ctxt->b & 7);
3559 3560 3561 3562
			break;
		case 0x40 ... 0x4f: /* REX */
			if (mode != X86EMUL_MODE_PROT64)
				goto done_prefixes;
3563
			ctxt->rex_prefix = ctxt->b;
3564 3565
			continue;
		case 0xf0:	/* LOCK */
3566
			ctxt->lock_prefix = 1;
3567 3568 3569
			break;
		case 0xf2:	/* REPNE/REPNZ */
		case 0xf3:	/* REP/REPE/REPZ */
3570
			ctxt->rep_prefix = ctxt->b;
3571 3572 3573 3574 3575 3576 3577
			break;
		default:
			goto done_prefixes;
		}

		/* Any legacy prefix after a REX prefix nullifies its effect. */

3578
		ctxt->rex_prefix = 0;
3579 3580 3581 3582 3583
	}

done_prefixes:

	/* REX prefix. */
3584 3585
	if (ctxt->rex_prefix & 8)
		ctxt->op_bytes = 8;	/* REX.W */
3586 3587

	/* Opcode byte(s). */
3588
	opcode = opcode_table[ctxt->b];
3589
	/* Two-byte opcode? */
3590 3591
	if (ctxt->b == 0x0f) {
		ctxt->twobyte = 1;
3592
		ctxt->b = insn_fetch(u8, ctxt);
3593
		opcode = twobyte_table[ctxt->b];
3594
	}
3595
	ctxt->d = opcode.flags;
3596

3597 3598
	while (ctxt->d & GroupMask) {
		switch (ctxt->d & GroupMask) {
3599
		case Group:
3600
			ctxt->modrm = insn_fetch(u8, ctxt);
3601 3602
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
3603 3604 3605
			opcode = opcode.u.group[goffset];
			break;
		case GroupDual:
3606
			ctxt->modrm = insn_fetch(u8, ctxt);
3607 3608 3609
			--ctxt->_eip;
			goffset = (ctxt->modrm >> 3) & 7;
			if ((ctxt->modrm >> 6) == 3)
3610 3611 3612 3613 3614
				opcode = opcode.u.gdual->mod3[goffset];
			else
				opcode = opcode.u.gdual->mod012[goffset];
			break;
		case RMExt:
3615
			goffset = ctxt->modrm & 7;
3616
			opcode = opcode.u.group[goffset];
3617 3618
			break;
		case Prefix:
3619
			if (ctxt->rep_prefix && op_prefix)
3620
				return EMULATION_FAILED;
3621
			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3622 3623 3624 3625 3626 3627 3628 3629
			switch (simd_prefix) {
			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
			}
			break;
		default:
3630
			return EMULATION_FAILED;
3631
		}
3632

3633
		ctxt->d &= ~(u64)GroupMask;
3634
		ctxt->d |= opcode.flags;
3635 3636
	}

3637 3638 3639
	ctxt->execute = opcode.u.execute;
	ctxt->check_perm = opcode.check_perm;
	ctxt->intercept = opcode.intercept;
3640 3641

	/* Unrecognised? */
3642
	if (ctxt->d == 0 || (ctxt->d & Undefined))
3643
		return EMULATION_FAILED;
3644

3645
	if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3646
		return EMULATION_FAILED;
3647

3648 3649
	if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
		ctxt->op_bytes = 8;
3650

3651
	if (ctxt->d & Op3264) {
3652
		if (mode == X86EMUL_MODE_PROT64)
3653
			ctxt->op_bytes = 8;
3654
		else
3655
			ctxt->op_bytes = 4;
3656 3657
	}

3658 3659
	if (ctxt->d & Sse)
		ctxt->op_bytes = 16;
Avi Kivity's avatar
Avi Kivity committed
3660

3661
	/* ModRM and SIB bytes. */
3662
	if (ctxt->d & ModRM) {
3663
		rc = decode_modrm(ctxt, &ctxt->memop);
3664 3665 3666
		if (!ctxt->has_seg_override)
			set_seg_override(ctxt, ctxt->modrm_seg);
	} else if (ctxt->d & MemAbs)
3667
		rc = decode_abs(ctxt, &ctxt->memop);
3668 3669 3670
	if (rc != X86EMUL_CONTINUE)
		goto done;

3671 3672
	if (!ctxt->has_seg_override)
		set_seg_override(ctxt, VCPU_SREG_DS);
3673

3674
	ctxt->memop.addr.mem.seg = seg_override(ctxt);
3675

3676 3677
	if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3678 3679 3680 3681 3682

	/*
	 * Decode and fetch the source operand: register, memory
	 * or immediate.
	 */
3683
	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3684 3685 3686
	if (rc != X86EMUL_CONTINUE)
		goto done;

3687 3688 3689 3690
	/*
	 * Decode and fetch the second source operand: register, memory
	 * or immediate.
	 */
3691
	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3692 3693 3694
	if (rc != X86EMUL_CONTINUE)
		goto done;

3695
	/* Decode and fetch the destination operand: register or memory. */
3696
	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3697 3698

done:
3699 3700
	if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
		ctxt->memopp->addr.mem.ea += ctxt->_eip;
3701

3702
	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3703 3704
}

3705 3706 3707 3708 3709
bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
{
	return ctxt->d & PageTable;
}

3710 3711 3712 3713 3714 3715 3716 3717 3718
static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
{
	/* The second termination condition only applies for REPE
	 * and REPNE. Test if the repeat string operation prefix is
	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
	 * corresponding termination condition according to:
	 * 	- if REPE/REPZ and ZF = 0 then done
	 * 	- if REPNE/REPNZ and ZF = 1 then done
	 */
3719 3720 3721
	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
3722
		 ((ctxt->eflags & EFLG_ZF) == 0))
3723
		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
3724 3725 3726 3727 3728 3729
		    ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
		return true;

	return false;
}

3730
int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3731
{
3732
	struct x86_emulate_ops *ops = ctxt->ops;
3733
	u64 msr_data;
3734
	int rc = X86EMUL_CONTINUE;
3735
	int saved_dst_type = ctxt->dst.type;
3736

3737
	ctxt->mem_read.pos = 0;
3738

3739
	if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3740
		rc = emulate_ud(ctxt);
3741 3742 3743
		goto done;
	}

3744
	/* LOCK prefix is allowed only with some instructions */
3745
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3746
		rc = emulate_ud(ctxt);
3747 3748 3749
		goto done;
	}

3750
	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3751
		rc = emulate_ud(ctxt);
3752 3753 3754
		goto done;
	}

3755
	if ((ctxt->d & Sse)
3756 3757
	    && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
		|| !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
Avi Kivity's avatar
Avi Kivity committed
3758 3759 3760 3761
		rc = emulate_ud(ctxt);
		goto done;
	}

3762
	if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
Avi Kivity's avatar
Avi Kivity committed
3763 3764 3765 3766
		rc = emulate_nm(ctxt);
		goto done;
	}

3767 3768
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3769
					      X86_ICPT_PRE_EXCEPT);
3770 3771 3772 3773
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3774
	/* Privileged instruction can be executed only in CPL=0 */
3775
	if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3776
		rc = emulate_gp(ctxt, 0);
3777 3778 3779
		goto done;
	}

3780
	/* Instruction can only be executed in protected mode */
3781
	if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3782 3783 3784 3785
		rc = emulate_ud(ctxt);
		goto done;
	}

3786
	/* Do instruction specific permission checks */
3787 3788
	if (ctxt->check_perm) {
		rc = ctxt->check_perm(ctxt);
3789 3790 3791 3792
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3793 3794
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3795
					      X86_ICPT_POST_EXCEPT);
3796 3797 3798 3799
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3800
	if (ctxt->rep_prefix && (ctxt->d & String)) {
3801
		/* All REP prefixes have the same first termination condition */
3802 3803
		if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
			ctxt->eip = ctxt->_eip;
3804 3805 3806 3807
			goto done;
		}
	}

3808 3809 3810
	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
3811
		if (rc != X86EMUL_CONTINUE)
3812
			goto done;
3813
		ctxt->src.orig_val64 = ctxt->src.val64;
3814 3815
	}

3816 3817 3818
	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
3819 3820 3821 3822
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3823
	if ((ctxt->d & DstMask) == ImplicitOps)
3824 3825 3826
		goto special_insn;


3827
	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3828
		/* optimisation - avoid slow emulated read if Mov */
3829 3830
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
3831 3832
		if (rc != X86EMUL_CONTINUE)
			goto done;
3833
	}
3834
	ctxt->dst.orig_val = ctxt->dst.val;
3835

3836 3837
special_insn:

3838 3839
	if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
3840
					      X86_ICPT_POST_MEMACCESS);
3841 3842 3843 3844
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

3845 3846
	if (ctxt->execute) {
		rc = ctxt->execute(ctxt);
3847 3848 3849 3850 3851
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

3852
	if (ctxt->twobyte)
3853 3854
		goto twobyte_insn;

3855
	switch (ctxt->b) {
3856
	case 0x40 ... 0x47: /* inc r16/r32 */
3857
		emulate_1op(ctxt, "inc");
3858 3859
		break;
	case 0x48 ... 0x4f: /* dec r16/r32 */
3860
		emulate_1op(ctxt, "dec");
3861
		break;
3862
	case 0x63:		/* movsxd */
3863
		if (ctxt->mode != X86EMUL_MODE_PROT64)
3864
			goto cannot_emulate;
3865
		ctxt->dst.val = (s32) ctxt->src.val;
3866
		break;
3867 3868
	case 0x6c:		/* insb */
	case 0x6d:		/* insw/insd */
3869
		ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3870
		goto do_io_in;
3871 3872
	case 0x6e:		/* outsb */
	case 0x6f:		/* outsw/outsd */
3873
		ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3874
		goto do_io_out;
3875
		break;
3876
	case 0x70 ... 0x7f: /* jcc (short) */
3877 3878
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
3879
		break;
Nitin A Kamble's avatar
Nitin A Kamble committed
3880
	case 0x8d: /* lea r16/r32, m */
3881
		ctxt->dst.val = ctxt->src.addr.mem.ea;
Nitin A Kamble's avatar
Nitin A Kamble committed
3882
		break;
3883
	case 0x8f:		/* pop (sole member of Grp1a) */
3884
		rc = em_grp1a(ctxt);
3885
		break;
3886
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
3887
		if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3888
			break;
3889 3890
		rc = em_xchg(ctxt);
		break;
3891
	case 0x98: /* cbw/cwde/cdqe */
3892 3893 3894 3895
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3896 3897
		}
		break;
3898
	case 0xc0 ... 0xc1:
3899
		rc = em_grp2(ctxt);
3900
		break;
3901
	case 0xcc:		/* int3 */
3902 3903
		rc = emulate_int(ctxt, 3);
		break;
3904
	case 0xcd:		/* int n */
3905
		rc = emulate_int(ctxt, ctxt->src.val);
3906 3907
		break;
	case 0xce:		/* into */
3908 3909
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
3910
		break;
3911
	case 0xd0 ... 0xd1:	/* Grp2 */
3912
		rc = em_grp2(ctxt);
3913 3914
		break;
	case 0xd2 ... 0xd3:	/* Grp2 */
3915
		ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3916
		rc = em_grp2(ctxt);
3917
		break;
3918 3919
	case 0xe4: 	/* inb */
	case 0xe5: 	/* in */
3920
		goto do_io_in;
3921 3922
	case 0xe6: /* outb */
	case 0xe7: /* out */
3923
		goto do_io_out;
3924
	case 0xe8: /* call (near) */ {
3925 3926 3927
		long int rel = ctxt->src.val;
		ctxt->src.val = (unsigned long) ctxt->_eip;
		jmp_rel(ctxt, rel);
3928
		rc = em_push(ctxt);
3929
		break;
3930 3931
	}
	case 0xe9: /* jmp rel */
3932
	case 0xeb: /* jmp rel short */
3933 3934
		jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
3935
		break;
3936 3937
	case 0xec: /* in al,dx */
	case 0xed: /* in (e/r)ax,dx */
3938
	do_io_in:
3939 3940
		if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
				     &ctxt->dst.val))
3941 3942
			goto done; /* IO is needed */
		break;
3943 3944
	case 0xee: /* out dx,al */
	case 0xef: /* out dx,(e/r)ax */
3945
	do_io_out:
3946 3947 3948
		ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
				      &ctxt->src.val, 1);
		ctxt->dst.type = OP_NONE;	/* Disable writeback. */
3949
		break;
3950
	case 0xf4:              /* hlt */
3951
		ctxt->ops->halt(ctxt);
3952
		break;
3953 3954 3955 3956 3957 3958 3959
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
3960 3961 3962
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
3963 3964 3965 3966 3967 3968
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
3969
	case 0xfe: /* Grp4 */
3970
		rc = em_grp45(ctxt);
3971
		break;
3972
	case 0xff: /* Grp5 */
3973 3974
		rc = em_grp45(ctxt);
		break;
3975 3976
	default:
		goto cannot_emulate;
3977
	}
3978

3979 3980 3981
	if (rc != X86EMUL_CONTINUE)
		goto done;

3982
writeback:
3983
	rc = writeback(ctxt);
3984
	if (rc != X86EMUL_CONTINUE)
3985 3986
		goto done;

3987 3988 3989 3990
	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
3991
	ctxt->dst.type = saved_dst_type;
3992

3993 3994 3995
	if ((ctxt->d & SrcMask) == SrcSI)
		string_addr_inc(ctxt, seg_override(ctxt),
				VCPU_REGS_RSI, &ctxt->src);
3996

3997
	if ((ctxt->d & DstMask) == DstDI)
3998
		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3999
				&ctxt->dst);
4000

4001 4002 4003
	if (ctxt->rep_prefix && (ctxt->d & String)) {
		struct read_cache *r = &ctxt->io_read;
		register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4004

4005 4006 4007 4008 4009
		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
4010
			if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4011 4012 4013 4014 4015 4016
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
4017
				ctxt->mem_read.end = 0;
4018 4019 4020
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
4021
		}
4022
	}
4023

4024
	ctxt->eip = ctxt->_eip;
4025 4026

done:
4027 4028
	if (rc == X86EMUL_PROPAGATE_FAULT)
		ctxt->have_exception = true;
4029 4030 4031
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

4032
	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4033 4034

twobyte_insn:
4035
	switch (ctxt->b) {
4036
	case 0x09:		/* wbinvd */
4037
		(ctxt->ops->wbinvd)(ctxt);
4038 4039
		break;
	case 0x08:		/* invd */
4040 4041 4042 4043
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
		break;
	case 0x20: /* mov cr, reg */
4044
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4045
		break;
4046
	case 0x21: /* mov from dr to reg */
4047
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4048
		break;
4049
	case 0x22: /* mov reg, cr */
4050
		if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4051
			emulate_gp(ctxt, 0);
4052
			rc = X86EMUL_PROPAGATE_FAULT;
4053 4054
			goto done;
		}
4055
		ctxt->dst.type = OP_NONE;
4056
		break;
4057
	case 0x23: /* mov from reg to dr */
4058
		if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4059
				((ctxt->mode == X86EMUL_MODE_PROT64) ?
4060
				 ~0ULL : ~0U)) < 0) {
4061
			/* #UD condition is already handled by the code above */
4062
			emulate_gp(ctxt, 0);
4063
			rc = X86EMUL_PROPAGATE_FAULT;
4064 4065 4066
			goto done;
		}

4067
		ctxt->dst.type = OP_NONE;	/* no writeback */
4068
		break;
4069 4070
	case 0x30:
		/* wrmsr */
4071 4072 4073
		msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
			| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
		if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4074
			emulate_gp(ctxt, 0);
4075
			rc = X86EMUL_PROPAGATE_FAULT;
4076
			goto done;
4077 4078 4079 4080 4081
		}
		rc = X86EMUL_CONTINUE;
		break;
	case 0x32:
		/* rdmsr */
4082
		if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4083
			emulate_gp(ctxt, 0);
4084
			rc = X86EMUL_PROPAGATE_FAULT;
4085
			goto done;
4086
		} else {
4087 4088
			ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
			ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4089 4090 4091
		}
		rc = X86EMUL_CONTINUE;
		break;
4092
	case 0x40 ... 0x4f:	/* cmov */
4093 4094 4095
		ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
		if (!test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.type = OP_NONE; /* no writeback */
4096
		break;
4097
	case 0x80 ... 0x8f: /* jnz rel, etc*/
4098 4099
		if (test_cc(ctxt->b, ctxt->eflags))
			jmp_rel(ctxt, ctxt->src.val);
4100
		break;
4101
	case 0x90 ... 0x9f:     /* setcc r/m8 */
4102
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4103
		break;
4104 4105
	case 0xa3:
	      bt:		/* bt */
4106
		ctxt->dst.type = OP_NONE;
4107
		/* only subword offset */
4108
		ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4109
		emulate_2op_SrcV_nobyte(ctxt, "bt");
4110
		break;
4111 4112
	case 0xa4: /* shld imm8, r, r/m */
	case 0xa5: /* shld cl, r, r/m */
4113
		emulate_2op_cl(ctxt, "shld");
4114
		break;
4115 4116
	case 0xab:
	      bts:		/* bts */
4117
		emulate_2op_SrcV_nobyte(ctxt, "bts");
4118
		break;
4119 4120
	case 0xac: /* shrd imm8, r, r/m */
	case 0xad: /* shrd cl, r, r/m */
4121
		emulate_2op_cl(ctxt, "shrd");
4122
		break;
4123 4124
	case 0xae:              /* clflush */
		break;
4125 4126 4127 4128 4129
	case 0xb0 ... 0xb1:	/* cmpxchg */
		/*
		 * Save real source value, then compare EAX against
		 * destination.
		 */
4130 4131
		ctxt->src.orig_val = ctxt->src.val;
		ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4132
		emulate_2op_SrcV(ctxt, "cmp");
4133
		if (ctxt->eflags & EFLG_ZF) {
4134
			/* Success: write back to memory. */
4135
			ctxt->dst.val = ctxt->src.orig_val;
4136 4137
		} else {
			/* Failure: write the value we saw to EAX. */
4138 4139
			ctxt->dst.type = OP_REG;
			ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4140 4141 4142 4143
		}
		break;
	case 0xb3:
	      btr:		/* btr */
4144
		emulate_2op_SrcV_nobyte(ctxt, "btr");
4145 4146
		break;
	case 0xb6 ... 0xb7:	/* movzx */
4147 4148 4149
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
						       : (u16) ctxt->src.val;
4150 4151
		break;
	case 0xba:		/* Grp8 */
4152
		switch (ctxt->modrm_reg & 3) {
4153 4154 4155 4156 4157 4158 4159 4160 4161 4162
		case 0:
			goto bt;
		case 1:
			goto bts;
		case 2:
			goto btr;
		case 3:
			goto btc;
		}
		break;
4163 4164
	case 0xbb:
	      btc:		/* btc */
4165
		emulate_2op_SrcV_nobyte(ctxt, "btc");
4166
		break;
4167 4168 4169
	case 0xbc: {		/* bsf */
		u8 zf;
		__asm__ ("bsf %2, %0; setz %1"
4170 4171
			 : "=r"(ctxt->dst.val), "=q"(zf)
			 : "r"(ctxt->src.val));
4172 4173 4174
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
4175
			ctxt->dst.type = OP_NONE;	/* Disable writeback. */
4176 4177 4178 4179 4180 4181
		}
		break;
	}
	case 0xbd: {		/* bsr */
		u8 zf;
		__asm__ ("bsr %2, %0; setz %1"
4182 4183
			 : "=r"(ctxt->dst.val), "=q"(zf)
			 : "r"(ctxt->src.val));
4184 4185 4186
		ctxt->eflags &= ~X86_EFLAGS_ZF;
		if (zf) {
			ctxt->eflags |= X86_EFLAGS_ZF;
4187
			ctxt->dst.type = OP_NONE;	/* Disable writeback. */
4188 4189 4190
		}
		break;
	}
4191
	case 0xbe ... 0xbf:	/* movsx */
4192 4193 4194
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
							(s16) ctxt->src.val;
4195
		break;
4196
	case 0xc0 ... 0xc1:	/* xadd */
4197
		emulate_2op_SrcV(ctxt, "add");
4198
		/* Write back the register source. */
4199 4200
		ctxt->src.val = ctxt->dst.orig_val;
		write_register_operand(&ctxt->src);
4201
		break;
4202
	case 0xc3:		/* movnti */
4203 4204 4205
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
							(u64) ctxt->src.val;
4206
		break;
4207
	case 0xc7:		/* Grp9 (cmpxchg8b) */
4208
		rc = em_grp9(ctxt);
4209
		break;
4210 4211
	default:
		goto cannot_emulate;
4212
	}
4213 4214 4215 4216

	if (rc != X86EMUL_CONTINUE)
		goto done;

4217 4218 4219
	goto writeback;

cannot_emulate:
4220
	return EMULATION_FAILED;
4221
}