Home | History | Annotate | Line # | Download | only in sljit_src
sljitNativeX86_32.c revision 1.2.2.3
      1 /*	$NetBSD: sljitNativeX86_32.c,v 1.2.2.3 2014/08/20 00:04:25 tls Exp $	*/
      2 
      3 /*
      4  *    Stack-less Just-In-Time compiler
      5  *
      6  *    Copyright 2009-2012 Zoltan Herczeg (hzmester (at) freemail.hu). All rights reserved.
      7  *
      8  * Redistribution and use in source and binary forms, with or without modification, are
      9  * permitted provided that the following conditions are met:
     10  *
     11  *   1. Redistributions of source code must retain the above copyright notice, this list of
     12  *      conditions and the following disclaimer.
     13  *
     14  *   2. Redistributions in binary form must reproduce the above copyright notice, this list
     15  *      of conditions and the following disclaimer in the documentation and/or other materials
     16  *      provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
     19  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
     21  * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
     22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
     23  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
     24  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     25  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
     26  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 /* x86 32-bit arch dependent functions. */
     30 
     31 static sljit_si emit_do_imm(struct sljit_compiler *compiler, sljit_ub opcode, sljit_sw imm)
     32 {
     33 	sljit_ub *inst;
     34 
     35 	inst = (sljit_ub*)ensure_buf(compiler, 1 + 1 + sizeof(sljit_sw));
     36 	FAIL_IF(!inst);
     37 	INC_SIZE(1 + sizeof(sljit_sw));
     38 	*inst++ = opcode;
     39 	*(sljit_sw*)inst = imm;
     40 	return SLJIT_SUCCESS;
     41 }
     42 
     43 static sljit_ub* generate_far_jump_code(struct sljit_jump *jump, sljit_ub *code_ptr, sljit_si type)
     44 {
     45 	if (type == SLJIT_JUMP) {
     46 		*code_ptr++ = JMP_i32;
     47 		jump->addr++;
     48 	}
     49 	else if (type >= SLJIT_FAST_CALL) {
     50 		*code_ptr++ = CALL_i32;
     51 		jump->addr++;
     52 	}
     53 	else {
     54 		*code_ptr++ = GROUP_0F;
     55 		*code_ptr++ = get_jump_code(type);
     56 		jump->addr += 2;
     57 	}
     58 
     59 	if (jump->flags & JUMP_LABEL)
     60 		jump->flags |= PATCH_MW;
     61 	else
     62 		*(sljit_sw*)code_ptr = jump->u.target - (jump->addr + 4);
     63 	code_ptr += 4;
     64 
     65 	return code_ptr;
     66 }
     67 
     68 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_enter(struct sljit_compiler *compiler, sljit_si args, sljit_si scratches, sljit_si saveds, sljit_si local_size)
     69 {
     70 	sljit_si size;
     71 	sljit_si locals_offset;
     72 	sljit_ub *inst;
     73 
     74 	CHECK_ERROR();
     75 	check_sljit_emit_enter(compiler, args, scratches, saveds, local_size);
     76 
     77 	compiler->scratches = scratches;
     78 	compiler->saveds = saveds;
     79 	compiler->args = args;
     80 	compiler->flags_saved = 0;
     81 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
     82 	compiler->logical_local_size = local_size;
     83 #endif
     84 
     85 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
     86 	size = 1 + (saveds <= 3 ? saveds : 3) + (args > 0 ? (args * 2) : 0) + (args > 2 ? 2 : 0);
     87 #else
     88 	size = 1 + (saveds <= 3 ? saveds : 3) + (args > 0 ? (2 + args * 3) : 0);
     89 #endif
     90 	inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
     91 	FAIL_IF(!inst);
     92 
     93 	INC_SIZE(size);
     94 	PUSH_REG(reg_map[TMP_REG1]);
     95 #if !(defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
     96 	if (args > 0) {
     97 		*inst++ = MOV_r_rm;
     98 		*inst++ = MOD_REG | (reg_map[TMP_REG1] << 3) | 0x4 /* esp */;
     99 	}
    100 #endif
    101 	if (saveds > 2)
    102 		PUSH_REG(reg_map[SLJIT_SAVED_REG3]);
    103 	if (saveds > 1)
    104 		PUSH_REG(reg_map[SLJIT_SAVED_REG2]);
    105 	if (saveds > 0)
    106 		PUSH_REG(reg_map[SLJIT_SAVED_REG1]);
    107 
    108 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    109 	if (args > 0) {
    110 		*inst++ = MOV_r_rm;
    111 		*inst++ = MOD_REG | (reg_map[SLJIT_SAVED_REG1] << 3) | reg_map[SLJIT_SCRATCH_REG3];
    112 	}
    113 	if (args > 1) {
    114 		*inst++ = MOV_r_rm;
    115 		*inst++ = MOD_REG | (reg_map[SLJIT_SAVED_REG2] << 3) | reg_map[SLJIT_SCRATCH_REG2];
    116 	}
    117 	if (args > 2) {
    118 		*inst++ = MOV_r_rm;
    119 		*inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG3] << 3) | 0x4 /* esp */;
    120 		*inst++ = 0x24;
    121 		*inst++ = sizeof(sljit_sw) * (3 + 2); /* saveds >= 3 as well. */
    122 	}
    123 #else
    124 	if (args > 0) {
    125 		*inst++ = MOV_r_rm;
    126 		*inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG1] << 3) | reg_map[TMP_REG1];
    127 		*inst++ = sizeof(sljit_sw) * 2;
    128 	}
    129 	if (args > 1) {
    130 		*inst++ = MOV_r_rm;
    131 		*inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG2] << 3) | reg_map[TMP_REG1];
    132 		*inst++ = sizeof(sljit_sw) * 3;
    133 	}
    134 	if (args > 2) {
    135 		*inst++ = MOV_r_rm;
    136 		*inst++ = MOD_DISP8 | (reg_map[SLJIT_SAVED_REG3] << 3) | reg_map[TMP_REG1];
    137 		*inst++ = sizeof(sljit_sw) * 4;
    138 	}
    139 #endif
    140 
    141 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    142 	locals_offset = 2 * sizeof(sljit_uw);
    143 #else
    144 	SLJIT_COMPILE_ASSERT(FIXED_LOCALS_OFFSET >= 2 * sizeof(sljit_uw), require_at_least_two_words);
    145 	locals_offset = FIXED_LOCALS_OFFSET;
    146 #endif
    147 	compiler->scratches_start = locals_offset;
    148 	if (scratches > 3)
    149 		locals_offset += (scratches - 3) * sizeof(sljit_uw);
    150 	compiler->saveds_start = locals_offset;
    151 	if (saveds > 3)
    152 		locals_offset += (saveds - 3) * sizeof(sljit_uw);
    153 	compiler->locals_offset = locals_offset;
    154 #if defined(__APPLE__)
    155 	saveds = (2 + (saveds <= 3 ? saveds : 3)) * sizeof(sljit_uw);
    156 	local_size = ((locals_offset + saveds + local_size + 15) & ~15) - saveds;
    157 #else
    158 	local_size = locals_offset + ((local_size + sizeof(sljit_uw) - 1) & ~(sizeof(sljit_uw) - 1));
    159 #endif
    160 
    161 	compiler->local_size = local_size;
    162 #ifdef _WIN32
    163 	if (local_size > 1024) {
    164 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    165 		FAIL_IF(emit_do_imm(compiler, MOV_r_i32 + reg_map[SLJIT_SCRATCH_REG1], local_size));
    166 #else
    167 		local_size -= FIXED_LOCALS_OFFSET;
    168 		FAIL_IF(emit_do_imm(compiler, MOV_r_i32 + reg_map[SLJIT_SCRATCH_REG1], local_size));
    169 		FAIL_IF(emit_non_cum_binary(compiler, SUB_r_rm, SUB_rm_r, SUB, SUB_EAX_i32,
    170 			SLJIT_LOCALS_REG, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, FIXED_LOCALS_OFFSET));
    171 #endif
    172 		FAIL_IF(sljit_emit_ijump(compiler, SLJIT_CALL1, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_grow_stack)));
    173 	}
    174 #endif
    175 
    176 	SLJIT_ASSERT(local_size > 0);
    177 	return emit_non_cum_binary(compiler, SUB_r_rm, SUB_rm_r, SUB, SUB_EAX_i32,
    178 		SLJIT_LOCALS_REG, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, local_size);
    179 }
    180 
    181 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_context(struct sljit_compiler *compiler, sljit_si args, sljit_si scratches, sljit_si saveds, sljit_si local_size)
    182 {
    183 	sljit_si locals_offset;
    184 
    185 	CHECK_ERROR_VOID();
    186 	check_sljit_set_context(compiler, args, scratches, saveds, local_size);
    187 
    188 	compiler->scratches = scratches;
    189 	compiler->saveds = saveds;
    190 	compiler->args = args;
    191 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
    192 	compiler->logical_local_size = local_size;
    193 #endif
    194 
    195 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    196 	locals_offset = 2 * sizeof(sljit_uw);
    197 #else
    198 	locals_offset = FIXED_LOCALS_OFFSET;
    199 #endif
    200 	compiler->scratches_start = locals_offset;
    201 	if (scratches > 3)
    202 		locals_offset += (scratches - 3) * sizeof(sljit_uw);
    203 	compiler->saveds_start = locals_offset;
    204 	if (saveds > 3)
    205 		locals_offset += (saveds - 3) * sizeof(sljit_uw);
    206 	compiler->locals_offset = locals_offset;
    207 #if defined(__APPLE__)
    208 	saveds = (2 + (saveds <= 3 ? saveds : 3)) * sizeof(sljit_uw);
    209 	compiler->local_size = ((locals_offset + saveds + local_size + 15) & ~15) - saveds;
    210 #else
    211 	compiler->local_size = locals_offset + ((local_size + sizeof(sljit_uw) - 1) & ~(sizeof(sljit_uw) - 1));
    212 #endif
    213 }
    214 
    215 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_return(struct sljit_compiler *compiler, sljit_si op, sljit_si src, sljit_sw srcw)
    216 {
    217 	sljit_si size;
    218 	sljit_ub *inst;
    219 
    220 	CHECK_ERROR();
    221 	check_sljit_emit_return(compiler, op, src, srcw);
    222 	SLJIT_ASSERT(compiler->args >= 0);
    223 
    224 	compiler->flags_saved = 0;
    225 	FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
    226 
    227 	SLJIT_ASSERT(compiler->local_size > 0);
    228 	FAIL_IF(emit_cum_binary(compiler, ADD_r_rm, ADD_rm_r, ADD, ADD_EAX_i32,
    229 		SLJIT_LOCALS_REG, 0, SLJIT_LOCALS_REG, 0, SLJIT_IMM, compiler->local_size));
    230 
    231 	size = 2 + (compiler->saveds <= 3 ? compiler->saveds : 3);
    232 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    233 	if (compiler->args > 2)
    234 		size += 2;
    235 #else
    236 	if (compiler->args > 0)
    237 		size += 2;
    238 #endif
    239 	inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
    240 	FAIL_IF(!inst);
    241 
    242 	INC_SIZE(size);
    243 
    244 	if (compiler->saveds > 0)
    245 		POP_REG(reg_map[SLJIT_SAVED_REG1]);
    246 	if (compiler->saveds > 1)
    247 		POP_REG(reg_map[SLJIT_SAVED_REG2]);
    248 	if (compiler->saveds > 2)
    249 		POP_REG(reg_map[SLJIT_SAVED_REG3]);
    250 	POP_REG(reg_map[TMP_REG1]);
    251 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    252 	if (compiler->args > 2)
    253 		RET_I16(sizeof(sljit_sw));
    254 	else
    255 		RET();
    256 #else
    257 	RET();
    258 #endif
    259 
    260 	return SLJIT_SUCCESS;
    261 }
    262 
    263 /* --------------------------------------------------------------------- */
    264 /*  Operators                                                            */
    265 /* --------------------------------------------------------------------- */
    266 
    267 /* Size contains the flags as well. */
    268 static sljit_ub* emit_x86_instruction(struct sljit_compiler *compiler, sljit_si size,
    269 	/* The register or immediate operand. */
    270 	sljit_si a, sljit_sw imma,
    271 	/* The general operand (not immediate). */
    272 	sljit_si b, sljit_sw immb)
    273 {
    274 	sljit_ub *inst;
    275 	sljit_ub *buf_ptr;
    276 	sljit_si flags = size & ~0xf;
    277 	sljit_si inst_size;
    278 
    279 	/* Both cannot be switched on. */
    280 	SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
    281 	/* Size flags not allowed for typed instructions. */
    282 	SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
    283 	/* Both size flags cannot be switched on. */
    284 	SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
    285 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
    286 	/* SSE2 and immediate is not possible. */
    287 	SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2));
    288 	SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3)
    289 		&& (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66)
    290 		&& (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66));
    291 #endif
    292 
    293 	size &= 0xf;
    294 	inst_size = size;
    295 
    296 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
    297 	if (flags & (EX86_PREF_F2 | EX86_PREF_F3))
    298 		inst_size++;
    299 #endif
    300 	if (flags & EX86_PREF_66)
    301 		inst_size++;
    302 
    303 	/* Calculate size of b. */
    304 	inst_size += 1; /* mod r/m byte. */
    305 	if (b & SLJIT_MEM) {
    306 		if ((b & REG_MASK) == SLJIT_UNUSED)
    307 			inst_size += sizeof(sljit_sw);
    308 		else if (immb != 0 && !(b & OFFS_REG_MASK)) {
    309 			/* Immediate operand. */
    310 			if (immb <= 127 && immb >= -128)
    311 				inst_size += sizeof(sljit_sb);
    312 			else
    313 				inst_size += sizeof(sljit_sw);
    314 		}
    315 
    316 		if ((b & REG_MASK) == SLJIT_LOCALS_REG && !(b & OFFS_REG_MASK))
    317 			b |= TO_OFFS_REG(SLJIT_LOCALS_REG);
    318 
    319 		if ((b & OFFS_REG_MASK) != SLJIT_UNUSED)
    320 			inst_size += 1; /* SIB byte. */
    321 	}
    322 
    323 	/* Calculate size of a. */
    324 	if (a & SLJIT_IMM) {
    325 		if (flags & EX86_BIN_INS) {
    326 			if (imma <= 127 && imma >= -128) {
    327 				inst_size += 1;
    328 				flags |= EX86_BYTE_ARG;
    329 			} else
    330 				inst_size += 4;
    331 		}
    332 		else if (flags & EX86_SHIFT_INS) {
    333 			imma &= 0x1f;
    334 			if (imma != 1) {
    335 				inst_size ++;
    336 				flags |= EX86_BYTE_ARG;
    337 			}
    338 		} else if (flags & EX86_BYTE_ARG)
    339 			inst_size++;
    340 		else if (flags & EX86_HALF_ARG)
    341 			inst_size += sizeof(short);
    342 		else
    343 			inst_size += sizeof(sljit_sw);
    344 	}
    345 	else
    346 		SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
    347 
    348 	inst = (sljit_ub*)ensure_buf(compiler, 1 + inst_size);
    349 	PTR_FAIL_IF(!inst);
    350 
    351 	/* Encoding the byte. */
    352 	INC_SIZE(inst_size);
    353 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
    354 	if (flags & EX86_PREF_F2)
    355 		*inst++ = 0xf2;
    356 	if (flags & EX86_PREF_F3)
    357 		*inst++ = 0xf3;
    358 #endif
    359 	if (flags & EX86_PREF_66)
    360 		*inst++ = 0x66;
    361 
    362 	buf_ptr = inst + size;
    363 
    364 	/* Encode mod/rm byte. */
    365 	if (!(flags & EX86_SHIFT_INS)) {
    366 		if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM))
    367 			*inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
    368 
    369 		if ((a & SLJIT_IMM) || (a == 0))
    370 			*buf_ptr = 0;
    371 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
    372 		else if (!(flags & EX86_SSE2))
    373 			*buf_ptr = reg_map[a] << 3;
    374 		else
    375 			*buf_ptr = a << 3;
    376 #else
    377 		else
    378 			*buf_ptr = reg_map[a] << 3;
    379 #endif
    380 	}
    381 	else {
    382 		if (a & SLJIT_IMM) {
    383 			if (imma == 1)
    384 				*inst = GROUP_SHIFT_1;
    385 			else
    386 				*inst = GROUP_SHIFT_N;
    387 		} else
    388 			*inst = GROUP_SHIFT_CL;
    389 		*buf_ptr = 0;
    390 	}
    391 
    392 	if (!(b & SLJIT_MEM))
    393 #if (defined SLJIT_SSE2 && SLJIT_SSE2)
    394 		*buf_ptr++ |= MOD_REG + ((!(flags & EX86_SSE2)) ? reg_map[b] : b);
    395 #else
    396 		*buf_ptr++ |= MOD_REG + reg_map[b];
    397 #endif
    398 	else if ((b & REG_MASK) != SLJIT_UNUSED) {
    399 		if ((b & OFFS_REG_MASK) == SLJIT_UNUSED || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_LOCALS_REG)) {
    400 			if (immb != 0) {
    401 				if (immb <= 127 && immb >= -128)
    402 					*buf_ptr |= 0x40;
    403 				else
    404 					*buf_ptr |= 0x80;
    405 			}
    406 
    407 			if ((b & OFFS_REG_MASK) == SLJIT_UNUSED)
    408 				*buf_ptr++ |= reg_map[b & REG_MASK];
    409 			else {
    410 				*buf_ptr++ |= 0x04;
    411 				*buf_ptr++ = reg_map[b & REG_MASK] | (reg_map[OFFS_REG(b)] << 3);
    412 			}
    413 
    414 			if (immb != 0) {
    415 				if (immb <= 127 && immb >= -128)
    416 					*buf_ptr++ = immb; /* 8 bit displacement. */
    417 				else {
    418 					*(sljit_sw*)buf_ptr = immb; /* 32 bit displacement. */
    419 					buf_ptr += sizeof(sljit_sw);
    420 				}
    421 			}
    422 		}
    423 		else {
    424 			*buf_ptr++ |= 0x04;
    425 			*buf_ptr++ = reg_map[b & REG_MASK] | (reg_map[OFFS_REG(b)] << 3) | (immb << 6);
    426 		}
    427 	}
    428 	else {
    429 		*buf_ptr++ |= 0x05;
    430 		*(sljit_sw*)buf_ptr = immb; /* 32 bit displacement. */
    431 		buf_ptr += sizeof(sljit_sw);
    432 	}
    433 
    434 	if (a & SLJIT_IMM) {
    435 		if (flags & EX86_BYTE_ARG)
    436 			*buf_ptr = imma;
    437 		else if (flags & EX86_HALF_ARG)
    438 			*(short*)buf_ptr = imma;
    439 		else if (!(flags & EX86_SHIFT_INS))
    440 			*(sljit_sw*)buf_ptr = imma;
    441 	}
    442 
    443 	return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1);
    444 }
    445 
    446 /* --------------------------------------------------------------------- */
    447 /*  Call / return instructions                                           */
    448 /* --------------------------------------------------------------------- */
    449 
    450 static SLJIT_INLINE sljit_si call_with_args(struct sljit_compiler *compiler, sljit_si type)
    451 {
    452 	sljit_ub *inst;
    453 
    454 #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL)
    455 	inst = (sljit_ub*)ensure_buf(compiler, type >= SLJIT_CALL3 ? 1 + 2 + 1 : 1 + 2);
    456 	FAIL_IF(!inst);
    457 	INC_SIZE(type >= SLJIT_CALL3 ? 2 + 1 : 2);
    458 
    459 	if (type >= SLJIT_CALL3)
    460 		PUSH_REG(reg_map[SLJIT_SCRATCH_REG3]);
    461 	*inst++ = MOV_r_rm;
    462 	*inst++ = MOD_REG | (reg_map[SLJIT_SCRATCH_REG3] << 3) | reg_map[SLJIT_SCRATCH_REG1];
    463 #else
    464 	inst = (sljit_ub*)ensure_buf(compiler, 1 + 4 * (type - SLJIT_CALL0));
    465 	FAIL_IF(!inst);
    466 	INC_SIZE(4 * (type - SLJIT_CALL0));
    467 
    468 	*inst++ = MOV_rm_r;
    469 	*inst++ = MOD_DISP8 | (reg_map[SLJIT_SCRATCH_REG1] << 3) | 0x4 /* SIB */;
    470 	*inst++ = (0x4 /* none*/ << 3) | reg_map[SLJIT_LOCALS_REG];
    471 	*inst++ = 0;
    472 	if (type >= SLJIT_CALL2) {
    473 		*inst++ = MOV_rm_r;
    474 		*inst++ = MOD_DISP8 | (reg_map[SLJIT_SCRATCH_REG2] << 3) | 0x4 /* SIB */;
    475 		*inst++ = (0x4 /* none*/ << 3) | reg_map[SLJIT_LOCALS_REG];
    476 		*inst++ = sizeof(sljit_sw);
    477 	}
    478 	if (type >= SLJIT_CALL3) {
    479 		*inst++ = MOV_rm_r;
    480 		*inst++ = MOD_DISP8 | (reg_map[SLJIT_SCRATCH_REG3] << 3) | 0x4 /* SIB */;
    481 		*inst++ = (0x4 /* none*/ << 3) | reg_map[SLJIT_LOCALS_REG];
    482 		*inst++ = 2 * sizeof(sljit_sw);
    483 	}
    484 #endif
    485 	return SLJIT_SUCCESS;
    486 }
    487 
    488 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_si dst, sljit_sw dstw)
    489 {
    490 	sljit_ub *inst;
    491 
    492 	CHECK_ERROR();
    493 	check_sljit_emit_fast_enter(compiler, dst, dstw);
    494 	ADJUST_LOCAL_OFFSET(dst, dstw);
    495 
    496 	CHECK_EXTRA_REGS(dst, dstw, (void)0);
    497 
    498 	/* For UNUSED dst. Uncommon, but possible. */
    499 	if (dst == SLJIT_UNUSED)
    500 		dst = TMP_REG1;
    501 
    502 	if (FAST_IS_REG(dst)) {
    503 		/* Unused dest is possible here. */
    504 		inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
    505 		FAIL_IF(!inst);
    506 
    507 		INC_SIZE(1);
    508 		POP_REG(reg_map[dst]);
    509 		return SLJIT_SUCCESS;
    510 	}
    511 
    512 	/* Memory. */
    513 	inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
    514 	FAIL_IF(!inst);
    515 	*inst++ = POP_rm;
    516 	return SLJIT_SUCCESS;
    517 }
    518 
    519 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_si src, sljit_sw srcw)
    520 {
    521 	sljit_ub *inst;
    522 
    523 	CHECK_ERROR();
    524 	check_sljit_emit_fast_return(compiler, src, srcw);
    525 	ADJUST_LOCAL_OFFSET(src, srcw);
    526 
    527 	CHECK_EXTRA_REGS(src, srcw, (void)0);
    528 
    529 	if (FAST_IS_REG(src)) {
    530 		inst = (sljit_ub*)ensure_buf(compiler, 1 + 1 + 1);
    531 		FAIL_IF(!inst);
    532 
    533 		INC_SIZE(1 + 1);
    534 		PUSH_REG(reg_map[src]);
    535 	}
    536 	else if (src & SLJIT_MEM) {
    537 		inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
    538 		FAIL_IF(!inst);
    539 		*inst++ = GROUP_FF;
    540 		*inst |= PUSH_rm;
    541 
    542 		inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
    543 		FAIL_IF(!inst);
    544 		INC_SIZE(1);
    545 	}
    546 	else {
    547 		/* SLJIT_IMM. */
    548 		inst = (sljit_ub*)ensure_buf(compiler, 1 + 5 + 1);
    549 		FAIL_IF(!inst);
    550 
    551 		INC_SIZE(5 + 1);
    552 		*inst++ = PUSH_i32;
    553 		*(sljit_sw*)inst = srcw;
    554 		inst += sizeof(sljit_sw);
    555 	}
    556 
    557 	RET();
    558 	return SLJIT_SUCCESS;
    559 }
    560