|
From: <sv...@va...> - 2012-04-20 23:58:27
|
sewardj 2012-04-21 00:58:17 +0100 (Sat, 21 Apr 2012)
New Revision: 2296
Log:
Merge branches/TCHAIN from r2271 (its creation point) into trunk.
Modified directories:
trunk/
Modified files:
trunk/Makefile-gcc
trunk/priv/guest_amd64_defs.h
trunk/priv/guest_amd64_helpers.c
trunk/priv/guest_amd64_toIR.c
trunk/priv/guest_arm_defs.h
trunk/priv/guest_arm_helpers.c
trunk/priv/guest_arm_toIR.c
trunk/priv/guest_generic_bb_to_IR.c
trunk/priv/guest_generic_bb_to_IR.h
trunk/priv/guest_generic_x87.c
trunk/priv/guest_generic_x87.h
trunk/priv/guest_ppc_defs.h
trunk/priv/guest_ppc_helpers.c
trunk/priv/guest_ppc_toIR.c
trunk/priv/guest_s390_defs.h
trunk/priv/guest_s390_helpers.c
trunk/priv/guest_s390_toIR.c
trunk/priv/guest_x86_defs.h
trunk/priv/guest_x86_helpers.c
trunk/priv/guest_x86_toIR.c
trunk/priv/host_amd64_defs.c
trunk/priv/host_amd64_defs.h
trunk/priv/host_amd64_isel.c
trunk/priv/host_arm_defs.c
trunk/priv/host_arm_defs.h
trunk/priv/host_arm_isel.c
trunk/priv/host_generic_reg_alloc2.c
trunk/priv/host_generic_regs.c
trunk/priv/host_generic_regs.h
trunk/priv/host_generic_simd64.c
trunk/priv/host_generic_simd64.h
trunk/priv/host_ppc_defs.c
trunk/priv/host_ppc_defs.h
trunk/priv/host_ppc_isel.c
trunk/priv/host_s390_defs.c
trunk/priv/host_s390_defs.h
trunk/priv/host_s390_isel.c
trunk/priv/host_x86_defs.c
trunk/priv/host_x86_defs.h
trunk/priv/host_x86_isel.c
trunk/priv/ir_defs.c
trunk/priv/ir_match.c
trunk/priv/ir_match.h
trunk/priv/ir_opt.c
trunk/priv/ir_opt.h
trunk/priv/main_globals.c
trunk/priv/main_globals.h
trunk/priv/main_main.c
trunk/priv/main_util.c
trunk/priv/main_util.h
trunk/pub/libvex.h
trunk/pub/libvex_guest_amd64.h
trunk/pub/libvex_guest_arm.h
trunk/pub/libvex_guest_ppc32.h
trunk/pub/libvex_guest_ppc64.h
trunk/pub/libvex_guest_s390x.h
trunk/pub/libvex_guest_x86.h
trunk/pub/libvex_ir.h
trunk/pub/libvex_s390x_common.h
trunk/pub/libvex_trc_values.h
trunk/switchback/switchback.c
trunk/test_main.c
trunk/test_main.h
Modified: trunk/
Modified: trunk/pub/libvex_s390x_common.h (+9 -4)
===================================================================
--- trunk/pub/libvex_s390x_common.h 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/pub/libvex_s390x_common.h 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -1,3 +1,4 @@
+/* -*- mode: C; c-basic-offset: 3; -*- */
/*--------------------------------------------------------------------*/
/*--- Common defs for s390x libvex_s390x_common.h ---*/
@@ -27,8 +28,6 @@
The GNU General Public License is contained in the file COPYING.
*/
-/* -*- mode: C; c-basic-offset: 3; -*- */
-
#ifndef __LIBVEX_PUB_S390X_H
#define __LIBVEX_PUB_S390X_H
@@ -42,7 +41,7 @@
/*--------------------------------------------------------------*/
#define S390_REGNO_RETURN_VALUE 2
-#define S390_REGNO_DISPATCH_CTR 12 /* Holds VG_(dispatch_ctr) */
+#define S390_REGNO_TCHAIN_SCRATCH 12
#define S390_REGNO_GUEST_STATE_POINTER 13
#define S390_REGNO_LINK_REGISTER 14
#define S390_REGNO_STACK_POINTER 15
@@ -52,7 +51,7 @@
/*--- Offsets in the stack frame allocated by the dispatcher ---*/
/*--------------------------------------------------------------*/
-/* Where the profiling dispatcher saves the r2 contents. */
+/* Where the dispatcher saves the r2 contents. */
#define S390_OFFSET_SAVED_R2 160+96
/* Where client's FPC register is saved. */
@@ -88,6 +87,12 @@
/* Number of double words needed to store all facility bits. */
#define S390_NUM_FACILITY_DW 2
+/* The length of the instructions issued by s390_tchain_load64 */
+#define S390_TCHAIN_LOAD64_LEN 16
+
+/* The length of the call insn (BASR) used in translation chaining */
+#define S390_TCHAIN_CALL_LEN 2
+
#endif /* __LIBVEX_PUB_S390X_H */
/*--------------------------------------------------------------------*/
Modified: trunk/pub/libvex_ir.h (+15 -3)
===================================================================
--- trunk/pub/libvex_ir.h 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/pub/libvex_ir.h 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -1712,8 +1712,9 @@
guest to restart a syscall that has been interrupted by a signal.
*/
typedef
- enum {
- Ijk_Boring=0x16000, /* not interesting; just goto next */
+ enum {
+ Ijk_INVALID=0x16000,
+ Ijk_Boring, /* not interesting; just goto next */
Ijk_Call, /* guest is doing a call */
Ijk_Ret, /* guest is doing a return */
Ijk_ClientReq, /* do guest client req before continuing */
@@ -2194,11 +2195,15 @@
/* Conditional exit from the middle of an IRSB.
ppIRStmt output: if (<guard>) goto {<jk>} <dst>
eg. if (t69) goto {Boring} 0x4000AAA:I32
+ If <guard> is true, the guest state is also updated by
+ PUT-ing <dst> at <offsIP>. This is done because a
+ taken exit must update the guest program counter.
*/
struct {
IRExpr* guard; /* Conditional expression */
IRJumpKind jk; /* Jump kind */
IRConst* dst; /* Jump target (constant only) */
+ Int offsIP; /* Guest state offset for IP */
} Exit;
} Ist;
}
@@ -2218,8 +2223,12 @@
IRExpr* addr, IRExpr* storedata );
extern IRStmt* IRStmt_Dirty ( IRDirty* details );
extern IRStmt* IRStmt_MBE ( IRMBusEvent event );
-extern IRStmt* IRStmt_Exit ( IRExpr* guard, IRJumpKind jk, IRConst* dst );
+extern IRStmt* IRStmt_Exit ( IRExpr* guard, IRJumpKind jk, IRConst* dst,
+ Int offsIP );
+// TEMP HACK
+#define IRStmt_Exit3(__guard,__jk,__dst) IRStmt_Exit(__guard,__jk,__dst,0)
+
/* Deep-copy an IRStmt. */
extern IRStmt* deepCopyIRStmt ( IRStmt* );
@@ -2263,6 +2272,8 @@
executes all the way to the end, without a side exit
- An indication of any special actions (JumpKind) needed
for this final jump.
+ - Offset of the IP field in the guest state. This will be
+ updated before the final jump is done.
"IRSB" stands for "IR Super Block".
*/
@@ -2274,6 +2285,7 @@
Int stmts_used;
IRExpr* next;
IRJumpKind jumpkind;
+ Int offsIP;
}
IRSB;
Modified: trunk/priv/host_s390_defs.h (+74 -11)
===================================================================
--- trunk/priv/host_s390_defs.h 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/priv/host_s390_defs.h 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -130,7 +130,6 @@
S390_INSN_TEST, /* test operand and set cc */
S390_INSN_CC2BOOL,/* convert condition code to 0/1 */
S390_INSN_COMPARE,
- S390_INSN_BRANCH, /* un/conditional goto */
S390_INSN_HELPER_CALL,
S390_INSN_CAS, /* compare and swap */
S390_INSN_BFP_BINOP, /* Binary floating point 32-bit / 64-bit */
@@ -144,7 +143,13 @@
S390_INSN_BFP128_CONVERT_FROM,
S390_INSN_MFENCE,
S390_INSN_GZERO, /* Assign zero to a guest register */
- S390_INSN_GADD /* Add a value to a guest register */
+ S390_INSN_GADD, /* Add a value to a guest register */
+ /* The following 5 insns are mandated by translation chaining */
+ S390_INSN_XDIRECT, /* direct transfer to guest address */
+ S390_INSN_XINDIR, /* indirect transfer to guest address */
+ S390_INSN_XASSISTED, /* assisted transfer to guest address */
+ S390_INSN_EVCHECK, /* Event check */
+ S390_INSN_PROFINC /* 64-bit profile counter increment */
} s390_insn_tag;
@@ -338,11 +343,6 @@
HReg op3;
HReg old_mem;
} cas;
- struct {
- IRJumpKind kind;
- s390_cc_t cond;
- s390_opnd_RMI dst;
- } branch;
/* Pseudo-insn for representing a helper call.
TARGET is the absolute address of the helper function
NUM_ARGS says how many arguments are being passed.
@@ -407,6 +407,44 @@
UChar delta;
ULong value; /* for debugging only */
} gadd;
+
+ /* The next 5 entries are generic to support translation chaining */
+
+ /* Update the guest IA value, then exit requesting to chain
+ to it. May be conditional. */
+ struct {
+ s390_cc_t cond;
+ Bool to_fast_entry; /* chain to the what entry point? */
+ Addr64 dst; /* next guest address */
+ s390_amode *guest_IA;
+ } xdirect;
+ /* Boring transfer to a guest address not known at JIT time.
+ Not chainable. May be conditional. */
+ struct {
+ s390_cc_t cond;
+ HReg dst;
+ s390_amode *guest_IA;
+ } xindir;
+ /* Assisted transfer to a guest address, most general case.
+ Not chainable. May be conditional. */
+ struct {
+ s390_cc_t cond;
+ IRJumpKind kind;
+ HReg dst;
+ s390_amode *guest_IA;
+ } xassisted;
+ struct {
+ /* fixs390: I don't think these are really needed
+ as the gsp and the offset are fixed no ? */
+ s390_amode *counter; /* dispatch counter */
+ s390_amode *fail_addr;
+ } evcheck;
+ struct {
+ /* No fields. The address of the counter to increment is
+ installed later, post-translation, by patching it in,
+ as it is not known at translation time. */
+ } profinc;
+
} variant;
} s390_insn;
@@ -433,7 +471,6 @@
s390_insn *s390_insn_test(UChar size, s390_opnd_RMI src);
s390_insn *s390_insn_compare(UChar size, HReg dst, s390_opnd_RMI opnd,
Bool signed_comparison);
-s390_insn *s390_insn_branch(IRJumpKind jk, s390_cc_t cond, s390_opnd_RMI dst);
s390_insn *s390_insn_helper_call(s390_cc_t cond, Addr64 target, UInt num_args,
HChar *name);
s390_insn *s390_insn_bfp_triop(UChar size, s390_bfp_triop_t, HReg dst, HReg op2,
@@ -460,6 +497,15 @@
s390_insn *s390_insn_gzero(UChar size, UInt offset);
s390_insn *s390_insn_gadd(UChar size, UInt offset, UChar delta, ULong value);
+/* Five for translation chaining */
+s390_insn *s390_insn_xdirect(s390_cc_t cond, Addr64 dst, s390_amode *guest_IA,
+ Bool to_fast_entry);
+s390_insn *s390_insn_xindir(s390_cc_t cond, HReg dst, s390_amode *guest_IA);
+s390_insn *s390_insn_xassisted(s390_cc_t cond, HReg dst, s390_amode *guest_IA,
+ IRJumpKind kind);
+s390_insn *s390_insn_evcheck(s390_amode *counter, s390_amode *fail_addr);
+s390_insn *s390_insn_profinc(void);
+
const HChar *s390_insn_as_string(const s390_insn *);
/*--------------------------------------------------------*/
@@ -475,14 +521,31 @@
void getRegUsage_S390Instr( HRegUsage *, s390_insn *, Bool );
void mapRegs_S390Instr ( HRegRemap *, s390_insn *, Bool );
Bool isMove_S390Instr ( s390_insn *, HReg *, HReg * );
-Int emit_S390Instr ( UChar *, Int, s390_insn *, Bool,
- void *, void * );
+Int emit_S390Instr ( Bool *, UChar *, Int, s390_insn *, Bool,
+ void *, void *, void *, void *);
void getAllocableRegs_S390( Int *, HReg **, Bool );
void genSpill_S390 ( HInstr **, HInstr **, HReg , Int , Bool );
void genReload_S390 ( HInstr **, HInstr **, HReg , Int , Bool );
s390_insn *directReload_S390 ( s390_insn *, HReg, Short );
-HInstrArray *iselSB_S390 ( IRSB *, VexArch, VexArchInfo *, VexAbiInfo * );
+HInstrArray *iselSB_S390 ( IRSB *, VexArch, VexArchInfo *, VexAbiInfo *,
+ Int, Int, Bool, Bool, Addr64);
+/* Return the number of bytes of code needed for an event check */
+Int evCheckSzB_S390(void);
+
+/* Perform a chaining and unchaining of an XDirect jump. */
+VexInvalRange chainXDirect_S390(void *place_to_chain,
+ void *disp_cp_chain_me_EXPECTED,
+ void *place_to_jump_to);
+
+VexInvalRange unchainXDirect_S390(void *place_to_unchain,
+ void *place_to_jump_to_EXPECTED,
+ void *disp_cp_chain_me);
+
+/* Patch the counter location into an existing ProfInc point. */
+VexInvalRange patchProfInc_S390(void *code_to_patch,
+ ULong *location_of_counter);
+
/* KLUDGE: See detailled comment in host_s390_defs.c. */
extern const VexArchInfo *s390_archinfo_host;
Modified: trunk/priv/host_s390_defs.c (+773 -146)
===================================================================
--- trunk/priv/host_s390_defs.c 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/priv/host_s390_defs.c 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -59,6 +59,7 @@
static Bool s390_insn_is_reg_reg_move(const s390_insn *, HReg *src, HReg *dst);
static void s390_insn_map_regs(HRegRemap *, s390_insn *);
static void s390_insn_get_reg_usage(HRegUsage *u, const s390_insn *);
+static UInt s390_tchain_load64_len(void);
/*------------------------------------------------------------*/
@@ -118,7 +119,7 @@
/* Total number of allocable registers (all classes) */
*nregs = 16 /* GPRs */
- 1 /* r0 */
- - 1 /* r12 register holding VG_(dispatch_ctr) */
+ - 1 /* r12 scratch register for translation chaining support */
- 1 /* r13 guest state pointer */
- 1 /* r14 link register */
- 1 /* r15 stack pointer */
@@ -144,12 +145,8 @@
Otherwise, they are available to the allocator */
(*arr)[i++] = mkHReg(10, HRcInt64, False);
(*arr)[i++] = mkHReg(11, HRcInt64, False);
- /* GPR12 is not available because it caches VG_(dispatch_ctr).
- Setting aside a register for the counter gives slightly better
- performance - most of the time. From the 10 tests in "make perf"
- 8 run faster with a max observed speedup of 2.6% for bz2. ffbench
- is the counter example. It runs 1.3% faster without the dedicated
- register. */
+ /* GPR12 is not available because it us used as a scratch register
+ in translation chaining. */
/* GPR13 is not available because it is used as guest state pointer */
/* GPR14 is not available because it is used as link register */
/* GPR15 is not available because it is used as stack pointer */
@@ -183,6 +180,7 @@
return mkHReg(S390_REGNO_GUEST_STATE_POINTER, HRcInt64, False);
}
+
/* Is VALUE within the domain of a 20-bit signed integer. */
static __inline__ Bool
fits_signed_20bit(Int value)
@@ -617,14 +615,6 @@
s390_opnd_RMI_get_reg_usage(u, insn->variant.compare.src2);
break;
- case S390_INSN_BRANCH:
- s390_opnd_RMI_get_reg_usage(u, insn->variant.branch.dst);
- /* The destination address is loaded into S390_REGNO_RETURN_VALUE.
- See s390_insn_branch_emit. */
- addHRegUse(u, HRmWrite,
- mkHReg(S390_REGNO_RETURN_VALUE, HRcInt64, False));
- break;
-
case S390_INSN_HELPER_CALL: {
UInt i;
@@ -718,6 +708,29 @@
case S390_INSN_GADD:
break;
+ case S390_INSN_EVCHECK:
+ s390_amode_get_reg_usage(u, insn->variant.evcheck.counter);
+ s390_amode_get_reg_usage(u, insn->variant.evcheck.fail_addr);
+ break;
+
+ case S390_INSN_PROFINC:
+ /* Does not use any register visible to the register allocator */
+ break;
+
+ case S390_INSN_XDIRECT:
+ s390_amode_get_reg_usage(u, insn->variant.xdirect.guest_IA);
+ break;
+
+ case S390_INSN_XINDIR:
+ addHRegUse(u, HRmRead, insn->variant.xindir.dst);
+ s390_amode_get_reg_usage(u, insn->variant.xindir.guest_IA);
+ break;
+
+ case S390_INSN_XASSISTED:
+ addHRegUse(u, HRmRead, insn->variant.xassisted.dst);
+ s390_amode_get_reg_usage(u, insn->variant.xassisted.guest_IA);
+ break;
+
default:
vpanic("s390_insn_get_reg_usage");
}
@@ -829,11 +842,6 @@
s390_opnd_RMI_map_regs(m, &insn->variant.compare.src2);
break;
- case S390_INSN_BRANCH:
- s390_opnd_RMI_map_regs(m, &insn->variant.branch.dst);
- /* No need to map S390_REGNO_RETURN_VALUE. It's not virtual */
- break;
-
case S390_INSN_HELPER_CALL:
/* s390_insn_helper_call_emit also reads / writes the link register
and stack pointer. But those registers are not visible to the
@@ -923,6 +931,31 @@
case S390_INSN_GADD:
break;
+ case S390_INSN_EVCHECK:
+ s390_amode_map_regs(m, insn->variant.evcheck.counter);
+ s390_amode_map_regs(m, insn->variant.evcheck.fail_addr);
+ break;
+
+ case S390_INSN_PROFINC:
+ /* Does not use any register visible to the register allocator */
+ break;
+
+ case S390_INSN_XDIRECT:
+ s390_amode_map_regs(m, insn->variant.xdirect.guest_IA);
+ break;
+
+ case S390_INSN_XINDIR:
+ s390_amode_map_regs(m, insn->variant.xindir.guest_IA);
+ insn->variant.xindir.dst =
+ lookupHRegRemap(m, insn->variant.xindir.dst);
+ break;
+
+ case S390_INSN_XASSISTED:
+ s390_amode_map_regs(m, insn->variant.xassisted.guest_IA);
+ insn->variant.xassisted.dst =
+ lookupHRegRemap(m, insn->variant.xassisted.dst);
+ break;
+
default:
vpanic("s390_insn_map_regs");
}
@@ -1403,6 +1436,16 @@
static UChar *
+s390_emit_BRCL(UChar *p, UChar r1, ULong i2)
+{
+ if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
+ s390_disasm(ENC2(XMNM, PCREL), S390_XMNM_BRCL, r1, i2);
+
+ return emit_RIL(p, 0xc00400000000ULL, r1, i2);
+}
+
+
+static UChar *
s390_emit_CR(UChar *p, UChar r1, UChar r2)
{
if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
@@ -4252,21 +4295,6 @@
s390_insn *
-s390_insn_branch(IRJumpKind kind, s390_cc_t cond, s390_opnd_RMI dst)
-{
- s390_insn *insn = LibVEX_Alloc(sizeof(s390_insn));
-
- insn->tag = S390_INSN_BRANCH;
- insn->size = 0; /* does not matter */
- insn->variant.branch.kind = kind;
- insn->variant.branch.dst = dst;
- insn->variant.branch.cond = cond;
-
- return insn;
-}
-
-
-s390_insn *
s390_insn_helper_call(s390_cc_t cond, Addr64 target, UInt num_args,
HChar *name)
{
@@ -4489,6 +4517,89 @@
}
+s390_insn *
+s390_insn_xdirect(s390_cc_t cond, Addr64 dst, s390_amode *guest_IA,
+ Bool to_fast_entry)
+{
+ s390_insn *insn = LibVEX_Alloc(sizeof(s390_insn));
+
+ insn->tag = S390_INSN_XDIRECT;
+ insn->size = 0; /* does not matter */
+
+ insn->variant.xdirect.cond = cond;
+ insn->variant.xdirect.dst = dst;
+ insn->variant.xdirect.guest_IA = guest_IA;
+ insn->variant.xdirect.to_fast_entry = to_fast_entry;
+
+ return insn;
+}
+
+
+s390_insn *
+s390_insn_xindir(s390_cc_t cond, HReg dst, s390_amode *guest_IA)
+{
+ s390_insn *insn = LibVEX_Alloc(sizeof(s390_insn));
+
+ insn->tag = S390_INSN_XINDIR;
+ insn->size = 0; /* does not matter */
+
+ insn->variant.xindir.cond = cond;
+ insn->variant.xindir.dst = dst;
+ insn->variant.xindir.guest_IA = guest_IA;
+
+ return insn;
+}
+
+
+s390_insn *
+s390_insn_xassisted(s390_cc_t cond, HReg dst, s390_amode *guest_IA,
+ IRJumpKind kind)
+{
+ s390_insn *insn = LibVEX_Alloc(sizeof(s390_insn));
+
+ insn->tag = S390_INSN_XASSISTED;
+ insn->size = 0; /* does not matter */
+
+ insn->variant.xassisted.cond = cond;
+ insn->variant.xassisted.dst = dst;
+ insn->variant.xassisted.guest_IA = guest_IA;
+ insn->variant.xassisted.kind = kind;
+
+ return insn;
+}
+
+
+s390_insn *
+s390_insn_evcheck(s390_amode *counter, s390_amode *fail_addr)
+{
+ s390_insn *insn = LibVEX_Alloc(sizeof(s390_insn));
+
+ vassert(counter->tag == S390_AMODE_B12 || counter->tag == S390_AMODE_BX12);
+ vassert(fail_addr->tag == S390_AMODE_B12 ||
+ fail_addr->tag == S390_AMODE_BX12);
+
+ insn->tag = S390_INSN_EVCHECK;
+ insn->size = 0; /* does not matter */
+
+ insn->variant.evcheck.counter = counter;
+ insn->variant.evcheck.fail_addr = fail_addr;
+
+ return insn;
+}
+
+
+s390_insn *
+s390_insn_profinc(void)
+{
+ s390_insn *insn = LibVEX_Alloc(sizeof(s390_insn));
+
+ insn->tag = S390_INSN_PROFINC;
+ insn->size = 0; /* does not matter */
+
+ return insn;
+}
+
+
/*---------------------------------------------------------------*/
/*--- Debug print ---*/
/*---------------------------------------------------------------*/
@@ -4792,11 +4903,6 @@
&insn->variant.compare.src2);
break;
- case S390_INSN_BRANCH:
- s390_sprintf(buf, "if (%C) %J %O", insn->variant.branch.cond,
- insn->variant.branch.kind, &insn->variant.branch.dst);
- return buf; /* avoid printing "size = ..." which is meaningless */
-
case S390_INSN_HELPER_CALL: {
s390_sprintf(buf, "%M if (%C) %s{%I}(%L)", "v-call",
insn->variant.helper_call.cond,
@@ -4924,6 +5030,39 @@
insn->variant.gadd.value);
break;
+ case S390_INSN_EVCHECK:
+ s390_sprintf(buf, "%M counter = %A, fail-addr = %A", "v-evcheck",
+ insn->variant.evcheck.counter,
+ insn->variant.evcheck.fail_addr);
+ return buf; /* avoid printing "size = ..." which is meaningless */
+
+ case S390_INSN_PROFINC:
+ s390_sprintf(buf, "%M", "v-profinc");
+ return buf; /* avoid printing "size = ..." which is meaningless */
+
+ case S390_INSN_XDIRECT:
+ s390_sprintf(buf, "%M if (%C) %A = %I %s", "v-xdirect",
+ insn->variant.xdirect.cond,
+ insn->variant.xdirect.guest_IA,
+ insn->variant.xdirect.dst,
+ insn->variant.xdirect.to_fast_entry ? "fast" : "slow");
+ return buf; /* avoid printing "size = ..." which is meaningless */
+
+ case S390_INSN_XINDIR:
+ s390_sprintf(buf, "%M if (%C) %A = %R", "v-xindir",
+ insn->variant.xindir.cond,
+ insn->variant.xindir.guest_IA,
+ insn->variant.xindir.dst);
+ return buf; /* avoid printing "size = ..." which is meaningless */
+
+ case S390_INSN_XASSISTED:
+ s390_sprintf(buf, "%M if (%C) %J %A = %R", "v-xassisted",
+ insn->variant.xassisted.cond,
+ insn->variant.xassisted.kind,
+ insn->variant.xassisted.guest_IA,
+ insn->variant.xassisted.dst);
+ return buf; /* avoid printing "size = ..." which is meaningless */
+
default: goto fail;
}
@@ -6512,104 +6651,6 @@
static UChar *
-s390_insn_branch_emit(UChar *buf, const s390_insn *insn)
-{
- s390_opnd_RMI dst;
- s390_cc_t cond;
- UInt trc;
- UChar *p, *ptmp = 0; /* avoid compiler warnings */
-
- cond = insn->variant.branch.cond;
- dst = insn->variant.branch.dst;
-
- p = buf;
- trc = 0;
-
- if (cond != S390_CC_ALWAYS) {
- /* So we have something like this
- if (cond) goto X;
- Y: ...
- We convert this into
- if (! cond) goto Y; // BRC insn; 4 bytes
- return_reg = X;
- return to dispatcher
- Y:
- */
- ptmp = p; /* 4 bytes (a BRC insn) to be filled in here */
- p += 4;
- }
-
- /* If a non-boring, set guest-state-pointer appropriately. */
-
- switch (insn->variant.branch.kind) {
- case Ijk_ClientReq: trc = VEX_TRC_JMP_CLIENTREQ; break;
- case Ijk_Sys_syscall: trc = VEX_TRC_JMP_SYS_SYSCALL; break;
- case Ijk_Yield: trc = VEX_TRC_JMP_YIELD; break;
- case Ijk_EmWarn: trc = VEX_TRC_JMP_EMWARN; break;
- case Ijk_EmFail: trc = VEX_TRC_JMP_EMFAIL; break;
- case Ijk_MapFail: trc = VEX_TRC_JMP_MAPFAIL; break;
- case Ijk_NoDecode: trc = VEX_TRC_JMP_NODECODE; break;
- case Ijk_TInval: trc = VEX_TRC_JMP_TINVAL; break;
- case Ijk_NoRedir: trc = VEX_TRC_JMP_NOREDIR; break;
- case Ijk_SigTRAP: trc = VEX_TRC_JMP_SIGTRAP; break;
- case Ijk_Ret: trc = 0; break;
- case Ijk_Call: trc = 0; break;
- case Ijk_Boring: trc = 0; break;
- break;
-
- default:
- vpanic("s390_insn_branch_emit: unknown jump kind");
- }
-
- /* Get the destination address into the return register */
- switch (dst.tag) {
- case S390_OPND_REG:
- p = s390_emit_LGR(p, S390_REGNO_RETURN_VALUE, hregNumber(dst.variant.reg));
- break;
-
- case S390_OPND_AMODE: {
- const s390_amode *am = dst.variant.am;
- UChar b = hregNumber(am->b);
- UChar x = hregNumber(am->x);
- Int d = am->d;
-
- p = s390_emit_LG(p, S390_REGNO_RETURN_VALUE, x, b, DISP20(d));
- break;
- }
-
- case S390_OPND_IMMEDIATE:
- p = s390_emit_load_64imm(p, S390_REGNO_RETURN_VALUE, dst.variant.imm);
- break;
-
- default:
- goto fail;
- }
-
- if (trc != 0) {
- /* Something special. Set guest-state pointer appropriately */
- p = s390_emit_LGHI(p, S390_REGNO_GUEST_STATE_POINTER, trc);
- } else {
- /* Nothing special needs to be done for calls and returns. */
- }
-
- p = s390_emit_BCR(p, S390_CC_ALWAYS, S390_REGNO_LINK_REGISTER);
-
- if (cond != S390_CC_ALWAYS) {
- Int delta = p - ptmp;
-
- delta >>= 1; /* immediate constant is #half-words */
- vassert(delta > 0 && delta < (1 << 16));
- s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
- }
-
- return p;
-
- fail:
- vpanic("s390_insn_branch_emit");
-}
-
-
-static UChar *
s390_insn_helper_call_emit(UChar *buf, const s390_insn *insn)
{
s390_cc_t cond;
@@ -7163,9 +7204,415 @@
}
+/* Define convenience functions needed for translation chaining.
+ Any changes need to be applied to the functions in concert. */
+
+static __inline__ Bool
+s390_insn_is_BRCL(const UChar *p, UChar condition)
+{
+ return p[0] == 0xc0 && p[1] == ((condition << 4) | 0x04);
+}
+
+static __inline__ Bool
+s390_insn_is_BR(const UChar *p, UChar reg)
+{
+ return p[0] == 0x07 && p[1] == (0xF0 | reg); /* BCR 15,reg */
+}
+
+static __inline__ Bool
+s390_insn_is_BASR(const UChar *p, UChar link_reg, UChar other_reg)
+{
+ return p[0] == 0x0D && p[1] == ((link_reg << 4) | other_reg);
+}
+
+/* Load the 64-bit VALUE into REG. Note that this function must NOT
+ optimise the generated code by looking at the value. I.e. using
+ LGHI if value == 0 would be very wrong.
+ fixs390: Do it in a way that works everywhere for now. */
+static UChar *
+s390_tchain_load64(UChar *buf, UChar regno, ULong value)
+{
+ UChar *begin = buf;
+
+ buf = s390_emit_IILL(buf, regno, value & 0xFFFF);
+ value >>= 16;
+ buf = s390_emit_IILH(buf, regno, value & 0xFFFF);
+ value >>= 16;
+ buf = s390_emit_IIHL(buf, regno, value & 0xFFFF);
+ value >>= 16;
+ buf = s390_emit_IIHH(buf, regno, value & 0xFFFF);
+
+ vassert(buf - begin == s390_tchain_load64_len());
+
+ return buf;
+}
+
+/* Return number of bytes generated by s390_tchain_load64 */
+static UInt
+s390_tchain_load64_len(void)
+{
+ return S390_TCHAIN_LOAD64_LEN;
+}
+
+/* Verify that CODE is the code sequence generated by s390_tchain_load64
+ to load VALUE into REGNO. Return pointer to the byte following the
+ insn sequence. */
+static const UChar *
+s390_tchain_verify_load64(const UChar *code, UChar regno, ULong value)
+{
+ UInt regmask = regno << 4;
+ UInt hw;
+
+ /* Check for IILL */
+ hw = value & 0xFFFF;
+ vassert(code[0] == 0xA5);
+ vassert(code[1] == (0x03 | regmask));
+ vassert(code[2] == (hw >> 8));
+ vassert(code[3] == (hw & 0xFF));
+
+ /* Check for IILH */
+ hw = (value >> 16) & 0xFFFF;
+ vassert(code[4] == 0xA5);
+ vassert(code[5] == (0x02 | regmask));
+ vassert(code[6] == (hw >> 8));
+ vassert(code[7] == (hw & 0xFF));
+
+ /* Check for IIHL */
+ hw = (value >> 32) & 0xFFFF;
+ vassert(code[8] == 0xA5);
+ vassert(code[9] == (0x01 | regmask));
+ vassert(code[10] == (hw >> 8));
+ vassert(code[11] == (hw & 0xFF));
+
+ /* Check for IIHH */
+ hw = (value >> 48) & 0xFFFF;
+ vassert(code[12] == 0xA5);
+ vassert(code[13] == (0x00 | regmask));
+ vassert(code[14] == (hw >> 8));
+ vassert(code[15] == (hw & 0xFF));
+
+ return code + s390_tchain_load64_len();
+}
+
+/* CODE points to the code sequence as generated by s390_tchain_load64.
+ Change the loaded value to VALUE. Return pointer to the byte following
+ the patched code sequence. */
+static UChar *
+s390_tchain_patch_load64(UChar *code, ULong imm64)
+{
+ code[3] = imm64 & 0xFF; imm64 >>= 8;
+ code[2] = imm64 & 0xFF; imm64 >>= 8;
+ code[7] = imm64 & 0xFF; imm64 >>= 8;
+ code[6] = imm64 & 0xFF; imm64 >>= 8;
+ code[11] = imm64 & 0xFF; imm64 >>= 8;
+ code[10] = imm64 & 0xFF; imm64 >>= 8;
+ code[15] = imm64 & 0xFF; imm64 >>= 8;
+ code[14] = imm64 & 0xFF; imm64 >>= 8;
+
+ return code + s390_tchain_load64_len();
+}
+
+
+/* NB: what goes on here has to be very closely coordinated with the
+ chainXDirect_S390 and unchainXDirect_S390 below. */
+static UChar *
+s390_insn_xdirect_emit(UChar *buf, const s390_insn *insn,
+ void *disp_cp_chain_me_to_slowEP,
+ void *disp_cp_chain_me_to_fastEP)
+{
+ /* We're generating chain-me requests here, so we need to be
+ sure this is actually allowed -- no-redir translations can't
+ use chain-me's. Hence: */
+ vassert(disp_cp_chain_me_to_slowEP != NULL);
+ vassert(disp_cp_chain_me_to_fastEP != NULL);
+
+ /* Use ptmp for backpatching conditional jumps. */
+ UChar *ptmp = buf;
+
+ /* First off, if this is conditional, create a conditional
+ jump over the rest of it. */
+ s390_cc_t cond = insn->variant.xdirect.cond;
+
+ if (cond != S390_CC_ALWAYS) {
+ /* So we have something like this
+ if (cond) do_xdirect;
+ Y: ...
+ We convert this into
+ if (! cond) goto Y; // BRC opcode; 4 bytes
+ do_xdirect;
+ Y:
+ */
+ /* 4 bytes (a BRC insn) to be filled in here */
+ buf += 4;
+ }
+
+ /* Update the guest IA. */
+ buf = s390_emit_load_64imm(buf, R0, insn->variant.xdirect.dst);
+
+ const s390_amode *amode = insn->variant.xdirect.guest_IA;
+ vassert(amode->tag == S390_AMODE_B12 || amode->tag == S390_AMODE_BX12);
+ UInt b = hregNumber(amode->b);
+ UInt x = hregNumber(amode->x); /* 0 for B12 and B20 */
+ UInt d = amode->d;
+
+ buf = s390_emit_STG(buf, R0, x, b, DISP20(d));
+
+ /* --- FIRST PATCHABLE BYTE follows --- */
+ /* VG_(disp_cp_chain_me_to_{slowEP,fastEP}) (where we're calling
+ to) backs up the return address, so as to find the address of
+ the first patchable byte. So: don't change the length of the
+ two instructions below. */
+
+ /* Load the chosen entry point into the scratch reg */
+ void *disp_cp_chain_me;
+
+ disp_cp_chain_me =
+ insn->variant.xdirect.to_fast_entry ? disp_cp_chain_me_to_fastEP
+ : disp_cp_chain_me_to_slowEP;
+
+ ULong addr = Ptr_to_ULong(disp_cp_chain_me);
+ buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH, addr);
+
+ /* call *tchain_scratch */
+ buf = s390_emit_BASR(buf, 1, S390_REGNO_TCHAIN_SCRATCH);
+
+ /* --- END of PATCHABLE BYTES --- */
+
+ /* Fix up the conditional jump, if there was one. */
+ if (cond != S390_CC_ALWAYS) {
+ Int delta = buf - ptmp;
+
+ delta >>= 1; /* immediate constant is #half-words */
+ vassert(delta > 0 && delta < (1 << 16));
+ s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
+ }
+
+ return buf;
+}
+
+/* Return the number of patchable bytes from an xdirect insn. */
+static UInt
+s390_xdirect_patchable_len(void)
+{
+ return s390_tchain_load64_len() + S390_TCHAIN_CALL_LEN;
+}
+
+
+static UChar *
+s390_insn_xindir_emit(UChar *buf, const s390_insn *insn, void *disp_cp_xindir)
+{
+ /* We're generating transfers that could lead indirectly to a
+ chain-me, so we need to be sure this is actually allowed --
+ no-redir translations are not allowed to reach normal
+ translations without going through the scheduler. That means
+ no XDirects or XIndirs out from no-redir translations.
+ Hence: */
+ vassert(disp_cp_xindir != NULL);
+
+ /* Use ptmp for backpatching conditional jumps. */
+ UChar *ptmp = buf;
+
+ /* First off, if this is conditional, create a conditional
+ jump over the rest of it. */
+ s390_cc_t cond = insn->variant.xdirect.cond;
+
+ if (cond != S390_CC_ALWAYS) {
+ /* So we have something like this
+ if (cond) do_xdirect;
+ Y: ...
+ We convert this into
+ if (! cond) goto Y; // BRC opcode; 4 bytes
+ do_xdirect;
+ Y:
+ */
+ /* 4 bytes (a BRC insn) to be filled in here */
+ buf += 4;
+ }
+
+ /* Update the guest IA with the address in xdirect.dst. */
+ const s390_amode *amode = insn->variant.xindir.guest_IA;
+
+ vassert(amode->tag == S390_AMODE_B12 || amode->tag == S390_AMODE_BX12);
+ UInt b = hregNumber(amode->b);
+ UInt x = hregNumber(amode->x); /* 0 for B12 and B20 */
+ UInt d = amode->d;
+ UInt regno = hregNumber(insn->variant.xindir.dst);
+
+ buf = s390_emit_STG(buf, regno, x, b, DISP20(d));
+
+ /* load tchain_scratch, #disp_indir */
+ buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH,
+ Ptr_to_ULong(disp_cp_xindir));
+ /* BR *tchain_direct */
+ buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
+
+ /* Fix up the conditional jump, if there was one. */
+ if (cond != S390_CC_ALWAYS) {
+ Int delta = buf - ptmp;
+
+ delta >>= 1; /* immediate constant is #half-words */
+ vassert(delta > 0 && delta < (1 << 16));
+ s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
+ }
+
+ return buf;
+}
+
+static UChar *
+s390_insn_xassisted_emit(UChar *buf, const s390_insn *insn,
+ void *disp_cp_xassisted)
+{
+ /* Use ptmp for backpatching conditional jumps. */
+ UChar *ptmp = buf;
+
+ /* First off, if this is conditional, create a conditional
+ jump over the rest of it. */
+ s390_cc_t cond = insn->variant.xdirect.cond;
+
+ if (cond != S390_CC_ALWAYS) {
+ /* So we have something like this
+ if (cond) do_xdirect;
+ Y: ...
+ We convert this into
+ if (! cond) goto Y; // BRC opcode; 4 bytes
+ do_xdirect;
+ Y:
+ */
+ /* 4 bytes (a BRC insn) to be filled in here */
+ buf += 4;
+ }
+
+ /* Update the guest IA with the address in xassisted.dst. */
+ const s390_amode *amode = insn->variant.xassisted.guest_IA;
+
+ vassert(amode->tag == S390_AMODE_B12 || amode->tag == S390_AMODE_BX12);
+ UInt b = hregNumber(amode->b);
+ UInt x = hregNumber(amode->x); /* 0 for B12 and B20 */
+ UInt d = amode->d;
+ UInt regno = hregNumber(insn->variant.xassisted.dst);
+
+ buf = s390_emit_STG(buf, regno, x, b, DISP20(d));
+
+ UInt trcval = 0;
+
+ switch (insn->variant.xassisted.kind) {
+ case Ijk_ClientReq: trcval = VEX_TRC_JMP_CLIENTREQ; break;
+ case Ijk_Sys_syscall: trcval = VEX_TRC_JMP_SYS_SYSCALL; break;
+ case Ijk_Yield: trcval = VEX_TRC_JMP_YIELD; break;
+ case Ijk_EmWarn: trcval = VEX_TRC_JMP_EMWARN; break;
+ case Ijk_MapFail: trcval = VEX_TRC_JMP_MAPFAIL; break;
+ case Ijk_NoDecode: trcval = VEX_TRC_JMP_NODECODE; break;
+ case Ijk_TInval: trcval = VEX_TRC_JMP_TINVAL; break;
+ case Ijk_NoRedir: trcval = VEX_TRC_JMP_NOREDIR; break;
+ case Ijk_SigTRAP: trcval = VEX_TRC_JMP_SIGTRAP; break;
+ case Ijk_SigSEGV: trcval = VEX_TRC_JMP_SIGSEGV; break;
+ case Ijk_Boring: trcval = VEX_TRC_JMP_BORING; break;
+ /* We don't expect to see the following being assisted. */
+ case Ijk_Ret:
+ case Ijk_Call:
+ /* fallthrough */
+ default:
+ ppIRJumpKind(insn->variant.xassisted.kind);
+ vpanic("s390_insn_xassisted_emit: unexpected jump kind");
+ }
+
+ vassert(trcval != 0);
+
+ /* guest_state_pointer = trcval */
+ buf = s390_emit_LGHI(buf, S390_REGNO_GUEST_STATE_POINTER, trcval);
+
+ /* load tchain_scratch, #disp_assisted */
+ buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH,
+ Ptr_to_ULong(disp_cp_xassisted));
+
+ /* BR *tchain_direct */
+ buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
+
+ /* Fix up the conditional jump, if there was one. */
+ if (cond != S390_CC_ALWAYS) {
+ Int delta = buf - ptmp;
+
+ delta >>= 1; /* immediate constant is #half-words */
+ vassert(delta > 0 && delta < (1 << 16));
+ s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
+ }
+
+ return buf;
+}
+
+
+/* Pseudo code:
+
+ guest_state[host_EvC_COUNTER] -= 1;
+ if (guest_state[host_EvC_COUNTER] >= 0) goto nofail;
+ goto guest_state[host_EvC_FAILADDR];
+ nofail: ;
+
+ The dispatch counter is a 32-bit value. */
+static UChar *
+s390_insn_evcheck_emit(UChar *buf, const s390_insn *insn)
+{
+ s390_amode *amode;
+ UInt b, x, d;
+ UChar *code_begin, *code_end;
+
+ code_begin = buf;
+
+ amode = insn->variant.evcheck.counter;
+ vassert(amode->tag == S390_AMODE_B12 || amode->tag == S390_AMODE_BX12);
+ b = hregNumber(amode->b);
+ x = hregNumber(amode->x); /* 0 for B12 and B20 */
+ d = amode->d;
+
+ /* Decrement the dispatch counter in the guest state */
+ /* fixs390: ASI if available */
+ buf = s390_emit_LHI(buf, R0, -1); /* 4 bytes */
+ buf = s390_emit_A(buf, R0, x, b, d); /* 4 bytes */
+ buf = s390_emit_ST(buf, R0, x, b, d); /* 4 bytes */
+
+ /* Jump over the next insn if >= 0 */
+ buf = s390_emit_BRC(buf, S390_CC_HE, (4 + 6 + 2) / 2); /* 4 bytes */
+
+ /* Computed goto to fail_address */
+ amode = insn->variant.evcheck.fail_addr;
+ b = hregNumber(amode->b);
+ x = hregNumber(amode->x); /* 0 for B12 and B20 */
+ d = amode->d;
+ buf = s390_emit_LG(buf, S390_REGNO_TCHAIN_SCRATCH, x, b, DISP20(d)); /* 6 bytes */
+ buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH); /* 2 bytes */
+
+ code_end = buf;
+
+ /* Make sure the size of the generated code is identical to the size
+ returned by evCheckSzB_S390 */
+ vassert(evCheckSzB_S390() == code_end - code_begin);
+
+ return buf;
+}
+
+
+static UChar *
+s390_insn_profinc_emit(UChar *buf,
+ const s390_insn *insn __attribute__((unused)))
+{
+ /* Generate a code template to increment a memory location whose
+ address will be known later as an immediate value. This code
+ template will be patched once the memory location is known.
+ For now we do this with address == 0. */
+ buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH, 0);
+ buf = s390_emit_LGHI(buf, R0, 1);
+ buf = s390_emit_AG( buf, R0, 0, S390_REGNO_TCHAIN_SCRATCH, DISP20(0));
+ buf = s390_emit_STG(buf, R0, 0, S390_REGNO_TCHAIN_SCRATCH, DISP20(0));
+
+ return buf;
+}
+
+
Int
-emit_S390Instr(UChar *buf, Int nbuf, s390_insn *insn, Bool mode64,
- void *dispatch_unassisted, void *dispatch_assisted)
+emit_S390Instr(Bool *is_profinc, UChar *buf, Int nbuf, s390_insn *insn,
+ Bool mode64, void *disp_cp_chain_me_to_slowEP,
+ void *disp_cp_chain_me_to_fastEP, void *disp_cp_xindir,
+ void *disp_cp_xassisted)
{
UChar *end;
@@ -7230,12 +7677,6 @@
end = s390_insn_compare_emit(buf, insn);
break;
- case S390_INSN_BRANCH:
- vassert(dispatch_unassisted == NULL);
- vassert(dispatch_assisted == NULL);
- end = s390_insn_branch_emit(buf, insn);
- break;
-
case S390_INSN_HELPER_CALL:
end = s390_insn_helper_call_emit(buf, insn);
break;
@@ -7288,6 +7729,30 @@
end = s390_insn_gadd_emit(buf, insn);
break;
+ case S390_INSN_PROFINC:
+ end = s390_insn_profinc_emit(buf, insn);
+ /* Tell the caller .. */
+ vassert(*is_profinc == False);
+ *is_profinc = True;
+ break;
+
+ case S390_INSN_EVCHECK:
+ end = s390_insn_evcheck_emit(buf, insn);
+ break;
+
+ case S390_INSN_XDIRECT:
+ end = s390_insn_xdirect_emit(buf, insn, disp_cp_chain_me_to_slowEP,
+ disp_cp_chain_me_to_fastEP);
+ break;
+
+ case S390_INSN_XINDIR:
+ end = s390_insn_xindir_emit(buf, insn, disp_cp_xindir);
+ break;
+
+ case S390_INSN_XASSISTED:
+ end = s390_insn_xassisted_emit(buf, insn, disp_cp_xassisted);
+ break;
+
default:
vpanic("emit_S390Instr");
}
@@ -7298,6 +7763,168 @@
}
+/* Return the number of bytes emitted for an S390_INSN_EVCHECK.
+ See s390_insn_evcheck_emit */
+Int
+evCheckSzB_S390(void)
+{
+ return 24;
+}
+
+
+/* Patch the counter address into CODE_TO_PATCH as previously
+ generated by s390_insn_profinc_emit. */
+VexInvalRange
+patchProfInc_S390(void *code_to_patch, ULong *location_of_counter)
+{
+ vassert(sizeof(ULong *) == 8);
+
+ s390_tchain_verify_load64(code_to_patch, S390_REGNO_TCHAIN_SCRATCH, 0);
+
+ s390_tchain_patch_load64(code_to_patch, Ptr_to_ULong(location_of_counter));
+
+ VexInvalRange vir = {0, 0};
+ return vir;
+}
+
+
+/* NB: what goes on here has to be very closely coordinated with the
+ s390_insn_xdirect_emit code above. */
+VexInvalRange
+chainXDirect_S390(void *place_to_chain,
+ void *disp_cp_chain_me_EXPECTED,
+ void *place_to_jump_to)
+{
+ /* What we're expecting to see @ PLACE_TI_CHAIN is:
+
+ load tchain-scratch, #disp_cp_chain_me_EXPECTED
+ BASR 1,S390_REGNO_TCHAIN_SCRATCH
+ */
+ const UChar *next;
+ next = s390_tchain_verify_load64(place_to_chain, S390_REGNO_TCHAIN_SCRATCH,
+ Ptr_to_ULong(disp_cp_chain_me_EXPECTED));
+ vassert(s390_insn_is_BASR(next, 1, S390_REGNO_TCHAIN_SCRATCH));
+
+ /* And what we want to change it to is either:
+ (general case):
+
+ load tchain_scratch, #place_to_jump_to
+ BR *tchain_scratch
+
+ ---OR---
+
+ in the case where the displacement is small enough
+
+ BRCL delta where delta is in half-words
+ invalid opcodes
+
+ In both cases the replacement has the same length as the original.
+ To remain sane & verifiable,
+ (1) limit the displacement for the short form to
+ (say) +/- one billion, so as to avoid wraparound
+ off-by-ones
+ (2) even if the short form is applicable, once every (say)
+ 1024 times use the long form anyway, so as to maintain
+ verifiability
+ */
+
+ /* This is the delta we need to put into a BRCL insn. Note, that the
+ offset in BRCL is in half-words. Hence division by 2. */
+ Long delta = (Long)((UChar *)place_to_jump_to - (UChar *)place_to_chain) / 2;
+ Bool shortOK = delta >= -1000*1000*1000 && delta < 1000*1000*1000;
+
+ static UInt shortCTR = 0; /* DO NOT MAKE NON-STATIC */
+ if (shortOK) {
+ shortCTR++; // thread safety bleh
+ if (0 == (shortCTR & 0x3FF)) {
+ shortOK = False;
+ if (0)
+ vex_printf("QQQ chainXDirect_S390: shortCTR = %u, "
+ "using long jmp\n", shortCTR);
+ }
+ }
+
+ /* And make the modifications. */
+ UChar *p = (UChar *)place_to_chain;
+ if (shortOK) {
+ p = s390_emit_BRCL(p, S390_CC_ALWAYS, delta); /* 6 bytes */
+
+ /* Make sure that BRCL fits into the patchable part of an xdirect
+ code sequence */
+ vassert(6 <= s390_xdirect_patchable_len());
+
+ /* Fill remaining bytes with 0x00 (invalid opcode) */
+ Int i;
+ for (i = 0; i < s390_xdirect_patchable_len() - 6; ++i)
+ p[i] = 0x00;
+ } else {
+ /*
+ load tchain_scratch, #place_to_jump_to
+ BR *tchain_scratch
+ */
+ ULong addr = Ptr_to_ULong(place_to_jump_to);
+ p = s390_tchain_load64(p, S390_REGNO_TCHAIN_SCRATCH, addr);
+ s390_emit_BCR(p, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
+ }
+
+ VexInvalRange vir = {0, 0};
+ return vir;
+}
+
+
+/* NB: what goes on here has to be very closely coordinated with the
+ s390_insn_xdirect_emit code above. */
+VexInvalRange
+unchainXDirect_S390(void *place_to_unchain,
+ void *place_to_jump_to_EXPECTED,
+ void *disp_cp_chain_me)
+{
+ /* What we're expecting to see @ PLACE_TO_UNCHAIN:
+
+ load tchain_scratch, #place_to_jump_to_EXPECTED
+ BR *tchain_scratch
+
+ ---OR---
+ in the case where the displacement falls within 32 bits
+
+ BRCL delta
+ invalid opcodes
+ */
+ UChar *p = place_to_unchain;
+
+ if (s390_insn_is_BRCL(p, S390_CC_ALWAYS)) {
+ /* Looks like the short form */
+ Int num_hw = *(Int *)&p[2];
+ Int delta = 2 *num_hw;
+
+ vassert(p + delta == place_to_jump_to_EXPECTED);
+
+ Int i;
+ for (i = 0; i < s390_xdirect_patchable_len() - 6; ++i)
+ vassert(p[6+i] == 0x00);
+ } else {
+ /* Should be the long form */
+ const UChar *next;
+
+ next = s390_tchain_verify_load64(p, S390_REGNO_TCHAIN_SCRATCH,
+ Ptr_to_ULong(place_to_jump_to_EXPECTED));
+ /* Check for BR *tchain_scratch */
+ vassert(s390_insn_is_BR(next, S390_REGNO_TCHAIN_SCRATCH));
+ }
+
+ /* And what we want to change it to is:
+
+ load tchain_scratch, #disp_cp_chain_me
+ call *tchain_scratch
+ */
+ ULong addr = Ptr_to_ULong(disp_cp_chain_me);
+ p = s390_tchain_load64(p, S390_REGNO_TCHAIN_SCRATCH, addr);
+ s390_emit_BASR(p, 1, S390_REGNO_TCHAIN_SCRATCH);
+
+ VexInvalRange vir = {0, 0};
+ return vir;
+}
+
/*---------------------------------------------------------------*/
/*--- end host_s390_defs.c ---*/
/*---------------------------------------------------------------*/
Property changed: trunk/priv/host_generic_regs.h (+0 -0)
___________________________________________________________________
Name: svn:mergeinfo
-
Property changed: trunk/priv/host_generic_regs.c (+0 -0)
___________________________________________________________________
Name: svn:mergeinfo
-
Modified: trunk/priv/guest_s390_helpers.c (+2 -0)
===================================================================
--- trunk/priv/guest_s390_helpers.c 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/priv/guest_s390_helpers.c 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -130,6 +130,8 @@
state->guest_TILEN = 0;
state->guest_IP_AT_SYSCALL = 0;
state->guest_EMWARN = EmWarn_NONE;
+ state->host_EvC_COUNTER = 0;
+ state->host_EvC_FAILADDR = 0;
/*------------------------------------------------------------*/
/*--- Initialise thunk ---*/
Modified: trunk/priv/guest_s390_defs.h (+0 -1)
===================================================================
--- trunk/priv/guest_s390_defs.h 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/priv/guest_s390_defs.h 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -43,7 +43,6 @@
/* Convert one s390 insn to IR. See the type DisOneInstrFn in
bb_to_IR.h. */
DisResult disInstr_S390 ( IRSB* irbb,
- Bool put_IP,
Bool (*resteerOkFn) ( void*, Addr64 ),
Bool resteerCisOk,
void* callback_opaque,
Modified: trunk/priv/host_s390_isel.c (+151 -18)
===================================================================
--- trunk/priv/host_s390_isel.c 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/priv/host_s390_isel.c 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -69,6 +69,18 @@
- The host subarchitecture we are selecting insns for.
This is set at the start and does not change.
+ - A Bool for indicating whether we may generate chain-me
+ instructions for control flow transfers, or whether we must use
+ XAssisted.
+
+ - The maximum guest address of any guest insn in this block.
+ Actually, the address of the highest-addressed byte from any insn
+ in this block. Is set at the start and does not change. This is
+ used for detecting jumps which are definitely forward-edges from
+ this block, and therefore can be made (chained) to the fast entry
+ point of the destination, thereby avoiding the destination's
+ event check.
+
- A flag to indicate whether the guest IA has been assigned to.
- Values of certain guest registers which are often assigned constants.
@@ -92,16 +104,19 @@
typedef struct {
IRTypeEnv *type_env;
+ HInstrArray *code;
HReg *vregmap;
HReg *vregmapHI;
UInt n_vregmap;
+ UInt vreg_ctr;
+ UInt hwcaps;
- HInstrArray *code;
-
ULong old_value[NUM_TRACKED_REGS];
- UInt vreg_ctr;
- UInt hwcaps;
+ /* The next two are for translation chaining */
+ Addr64 max_ga;
+ Bool chaining_allowed;
+
Bool first_IA_assignment;
Bool old_value_valid[NUM_TRACKED_REGS];
} ISelEnv;
@@ -2437,17 +2452,56 @@
/* --------- EXIT --------- */
case Ist_Exit: {
- s390_opnd_RMI dst;
s390_cc_t cond;
IRConstTag tag = stmt->Ist.Exit.dst->tag;
if (tag != Ico_U64)
vpanic("s390_isel_stmt: Ist_Exit: dst is not a 64-bit value");
- dst = s390_isel_int_expr_RMI(env, IRExpr_Const(stmt->Ist.Exit.dst));
+ s390_amode *guest_IA = s390_amode_for_guest_state(stmt->Ist.Exit.offsIP);
cond = s390_isel_cc(env, stmt->Ist.Exit.guard);
- addInstr(env, s390_insn_branch(stmt->Ist.Exit.jk, cond, dst));
- return;
+
+ /* Case: boring transfer to known address */
+ if (stmt->Ist.Exit.jk == Ijk_Boring) {
+ if (env->chaining_allowed) {
+ /* .. almost always true .. */
+ /* Skip the event check at the dst if this is a forwards
+ edge. */
+ Bool to_fast_entry
+ = ((Addr64)stmt->Ist.Exit.dst->Ico.U64) > env->max_ga;
+ if (0) vex_printf("%s", to_fast_entry ? "Y" : ",");
+ addInstr(env, s390_insn_xdirect(cond, stmt->Ist.Exit.dst->Ico.U64,
+ guest_IA, to_fast_entry));
+ } else {
+ /* .. very occasionally .. */
+ /* We can't use chaining, so ask for an assisted transfer,
+ as that's the only alternative that is allowable. */
+ HReg dst = s390_isel_int_expr(env,
+ IRExpr_Const(stmt->Ist.Exit.dst));
+ addInstr(env, s390_insn_xassisted(cond, dst, guest_IA, Ijk_Boring));
+ }
+ return;
+ }
+
+ /* Case: assisted transfer to arbitrary address */
+ switch (stmt->Ist.Exit.jk) {
+ case Ijk_TInval:
+ case Ijk_Sys_syscall:
+ case Ijk_ClientReq:
+ case Ijk_NoRedir:
+ case Ijk_Yield:
+ case Ijk_SigTRAP: {
+ HReg dst = s390_isel_int_expr(env, IRExpr_Const(stmt->Ist.Exit.dst));
+ addInstr(env, s390_insn_xassisted(cond, dst, guest_IA,
+ stmt->Ist.Exit.jk));
+ return;
+ }
+ default:
+ break;
+ }
+
+ /* Do we ever expect to see any other kind? */
+ goto stmt_fail;
}
/* --------- MEM FENCE --------- */
@@ -2484,20 +2538,80 @@
/*---------------------------------------------------------*/
static void
-iselNext(ISelEnv *env, IRExpr *next, IRJumpKind jk)
+iselNext(ISelEnv *env, IRExpr *next, IRJumpKind jk, int offsIP)
{
- s390_opnd_RMI dst;
-
if (vex_traceflags & VEX_TRACE_VCODE) {
- vex_printf("\n-- goto {");
+ vex_printf("\n-- PUT(%d) = ", offsIP);
+ ppIRExpr(next);
+ vex_printf("; exit-");
ppIRJumpKind(jk);
- vex_printf("} ");
- ppIRExpr(next);
vex_printf("\n");
}
- dst = s390_isel_int_expr_RMI(env, next);
- addInstr(env, s390_insn_branch(jk, S390_CC_ALWAYS, dst));
+ s390_amode *guest_IA = s390_amode_for_guest_state(offsIP);
+
+ /* Case: boring transfer to known address */
+ if (next->tag == Iex_Const) {
+ IRConst *cdst = next->Iex.Const.con;
+ vassert(cdst->tag == Ico_U64);
+ if (jk == Ijk_Boring || jk == Ijk_Call) {
+ /* Boring transfer to known address */
+ if (env->chaining_allowed) {
+ /* .. almost always true .. */
+ /* Skip the event check at the dst if this is a forwards
+ edge. */
+ Bool to_fast_entry
+ = ((Addr64)cdst->Ico.U64) > env->max_ga;
+ if (0) vex_printf("%s", to_fast_entry ? "X" : ".");
+ addInstr(env, s390_insn_xdirect(S390_CC_ALWAYS, cdst->Ico.U64,
+ guest_IA, to_fast_entry));
+ } else {
+ /* .. very occasionally .. */
+ /* We can't use chaining, so ask for an indirect transfer,
+ as that's the cheapest alternative that is allowable. */
+ HReg dst = s390_isel_int_expr(env, next);
+ addInstr(env, s390_insn_xassisted(S390_CC_ALWAYS, dst, guest_IA,
+ Ijk_Boring));
+ }
+ return;
+ }
+ }
+
+ /* Case: call/return (==boring) transfer to any address */
+ switch (jk) {
+ case Ijk_Boring:
+ case Ijk_Ret:
+ case Ijk_Call: {
+ HReg dst = s390_isel_int_expr(env, next);
+ if (env->chaining_allowed) {
+ addInstr(env, s390_insn_xindir(S390_CC_ALWAYS, dst, guest_IA));
+ } else {
+ addInstr(env, s390_insn_xassisted(S390_CC_ALWAYS, dst, guest_IA,
+ Ijk_Boring));
+ }
+ return;
+ }
+ default:
+ break;
+ }
+
+ /* Case: some other kind of transfer to any address */
+ switch (jk) {
+ case Ijk_TInval:
+ case Ijk_Sys_syscall:
+ case Ijk_ClientReq:
+ case Ijk_NoRedir:
+ case Ijk_Yield:
+ case Ijk_SigTRAP: {
+ HReg dst = s390_isel_int_expr(env, next);
+ addInstr(env, s390_insn_xassisted(S390_CC_ALWAYS, dst, guest_IA, jk));
+ return;
+ }
+ default:
+ break;
+ }
+
+ vpanic("iselNext");
}
@@ -2509,7 +2623,9 @@
HInstrArray *
iselSB_S390(IRSB *bb, VexArch arch_host, VexArchInfo *archinfo_host,
- VexAbiInfo *vbi)
+ VexAbiInfo *vbi, Int offset_host_evcheck_counter,
+ Int offset_host_evcheck_fail_addr, Bool chaining_allowed,
+ Bool add_profinc, Addr64 max_ga)
{
UInt i, j;
HReg hreg, hregHI;
@@ -2552,6 +2668,9 @@
/* and finally ... */
env->hwcaps = hwcaps_host;
+ env->max_ga = max_ga;
+ env->chaining_allowed = chaining_allowed;
+
/* For each IR temporary, allocate a suitably-kinded virtual
register. */
j = 0;
@@ -2595,12 +2714,26 @@
}
env->vreg_ctr = j;
+ /* The very first instruction must be an event check. */
+ s390_amode *counter, *fail_addr;
+ counter = s390_amode_for_guest_state(offset_host_evcheck_counter);
+ fail_addr = s390_amode_for_guest_state(offset_host_evcheck_fail_addr);
+ addInstr(env, s390_insn_evcheck(counter, fail_addr));
+
+ /* Possibly a block counter increment (for profiling). At this
+ point we don't know the address of the counter, so just pretend
+ it is zero. It will have to be patched later, but before this
+ translation is used, by a call to LibVEX_patchProfInc. */
+ if (add_profinc) {
+ addInstr(env, s390_insn_profinc());
+ }
+
/* Ok, finally we can iterate over the statements. */
for (i = 0; i < bb->stmts_used; i++)
if (bb->stmts[i])
s390_isel_stmt(env, bb->stmts[i]);
- iselNext(env, bb->next, bb->jumpkind);
+ iselNext(env, bb->next, bb->jumpkind, bb->offsIP);
/* Record the number of vregs we used. */
env->code->n_vregs = env->vreg_ctr;
Property changed: trunk/priv/guest_arm_toIR.c (+0 -0)
___________________________________________________________________
Name: svn:mergeinfo
-
Modified: trunk/priv/guest_arm_toIR.c (+162 -172)
===================================================================
--- trunk/priv/guest_arm_toIR.c 2012-04-20 23:33:44 +01:00 (rev 2295)
+++ trunk/priv/guest_arm_toIR.c 2012-04-21 00:58:17 -23:00 (rev 2296)
@@ -1398,7 +1398,8 @@
stmt( IRStmt_Exit(
unop(Iop_Not1, unop(Iop_32to1, mkexpr(guardT))),
Ijk_Boring,
- IRConst_U32(toUInt(guest_R15_curr_instr_notENC + 4))
+ IRConst_U32(toUInt(guest_R15_curr_instr_notENC + 4)),
+ OFFB_R15T
));
}
@@ -1414,7 +1415,8 @@
stmt( IRStmt_Exit(
unop(Iop_Not1, unop(Iop_32to1, mkexpr(guardT))),
Ijk_Boring,
- IRConst_U32(toUInt((guest_R15_curr_instr_notENC + 2) | 1))
+ IRConst_U32(toUInt((guest_R15_curr_instr_notENC + 2) | 1)),
+ OFFB_R15T
));
}
@@ -1431,7 +1433,8 @@
stmt( IRStmt_Exit(
unop(Iop_Not1, unop(Iop_32to1, mkexpr(guardT))),
Ijk_Boring,
- IRConst_U32(toUInt((guest_R15_curr_instr_notENC + 4) | 1))
+ IRConst_U32(toUInt((guest_R15_curr_instr_notENC + 4) | 1)),
+ OFFB_R15T
));
}
@@ -1448,7 +1451,8 @@
IRStmt_Exit(
binop(Iop_CmpNE32, mkexpr(t), mkU32(0)),
Ijk_NoDecode,
- IRConst_U32(toUInt(guest_R15_curr_instr_notENC | 1))
+ IRConst_U32(toUInt(guest_R15_curr_instr_notENC | 1)),
+ OFFB_R15T
)
);
}
@@ -11962,9 +11966,9 @@
UInt dst = guest_R15_curr_instr_notENC + 8 + (simm24 | 1);
putIRegA( 14, mkU32(guest_R15_curr_instr_notENC + 4),
IRTemp_INVALID/*because AL*/, Ijk_Boring );
- irsb->next = mkU32(dst);
- irsb->jumpkind = Ijk_Call;
- dres->whatNext = Dis_StopHere;
+ llPutIReg(15, mkU32(dst));
+ dres->jk_StopHere = Ijk_Call;
+ dres->whatNext = Dis_StopHere;
DIP("blx 0x%x (and switch to Thumb mode)\n", dst - 1);
return True;
}
@@ -12040,7 +12044,6 @@
static
DisResult disInstr_ARM_WRK (
- Bool put_IP,
Bool (*resteerOkFn) ( /*opaque*/void*, Addr64 ),
Bool resteerCisOk,
void* callback_opaque,
@@ -12066,9 +12069,10 @@
// etc etc
/* Set result defaults. */
- dres.whatNext = Dis_Continue;
- dres.len = 4;
- dres.continueAt = 0;
+ dres.whatNext = Dis_Continue;
+ dres.len = 4;
+ dres.continueAt = 0;
+ dres.jk_StopHere = Ijk_INVALID;
/* Set default actions for post-insn handling of writes to r15, if
required. */
@@ -12085,11 +12089,7 @@
DIP("\t(arm) 0x%x: ", (UInt)guest_R15_curr_instr_notENC);
- /* We may be asked to update the guest R15 before going further. */
vassert(0 == (guest_R15_curr_instr_notENC & 3));
- if (put_IP) {
- llPutIReg( 15, mkU32(guest_R15_curr_instr_notENC) );
- }
/* ----------------------------------------------------------- */
@@ -12116,9 +12116,9 @@
/* orr r10,r10,r10 */) {
/* R3 = client_request ( R4 ) */
DIP("r3 = client_request ( %%r4 )\n");
- irsb->next = mkU32( guest_R15_curr_instr_notENC + 20 );
- irsb->jumpkind = Ijk_ClientReq;
- dres.whatNext = Dis_StopHere;
+ llPutIReg(15, mkU32( guest_R15_curr_instr_notENC + 20 ));
+ dres.jk_StopHere = Ijk_ClientReq;
+ dres.whatNext = Dis_StopHere;
goto decode_success;
}
else
@@ -12136,9 +12136,9 @@
/* branch-and-link-to-noredir R4 */
DIP("branch-and-link-to-noredir r4\n");
llPutIReg(14, mkU32( guest_R15_curr_instr_notENC + 20) );
- irsb->next = llGetIReg(4);
- irsb->jumpkind = Ijk_NoRedir;
- dres.whatNext = Dis_StopHere;
+ llPutIReg(15, llGetIReg(4));
+ dres.jk_StopHere = Ijk_NoRedir;
+ dres.whatNext = Dis_StopHere;
goto decode_success;
}
/* We don't know what it is. Set opc1/opc2 so decode_failure
@@ -12977,9 +12977,9 @@
dres.continueA...
[truncated message content] |