|
From: <sv...@va...> - 2005-12-22 08:08:07
|
Author: njn
Date: 2005-12-22 08:08:01 +0000 (Thu, 22 Dec 2005)
New Revision: 5403
Log:
Clean up the secmap access functions a little more.
Modified:
branches/COMPVBITS/memcheck/mc_main.c
Modified: branches/COMPVBITS/memcheck/mc_main.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- branches/COMPVBITS/memcheck/mc_main.c 2005-12-22 06:20:59 UTC (rev 54=
02)
+++ branches/COMPVBITS/memcheck/mc_main.c 2005-12-22 08:08:01 UTC (rev 54=
03)
@@ -232,10 +232,14 @@
#define SM_OFF(aaa) (((aaa) & 0xffff) >> 2)
#define SM_OFF_64(aaa) (((aaa) & 0xffff) >> 3)
=20
-static inline Addr start_of_this_sm ( Addr a ) {
+// Paranoia: it's critical for performance that the requested inlining
+// occurs. So try extra hard.
+#define INLINE inline __attribute__((always_inline))
+
+static INLINE Addr start_of_this_sm ( Addr a ) {
return (a & (~SM_MASK));
}
-static inline Bool is_start_of_sm ( Addr a ) {
+static INLINE Bool is_start_of_sm ( Addr a ) {
return (start_of_this_sm(a) =3D=3D a);
}
=20
@@ -255,7 +259,7 @@
=20
static SecMap sm_distinguished[3];
=20
-static inline Bool is_distinguished_sm ( SecMap* sm ) {
+static INLINE Bool is_distinguished_sm ( SecMap* sm ) {
return sm >=3D &sm_distinguished[0] && sm <=3D &sm_distinguished[2];
}
=20
@@ -381,16 +385,57 @@
=20
/* --------------- SecMap fundamentals --------------- */
=20
-__attribute__((always_inline))
-static inline SecMap* get_secmap_readable_low ( Addr a )
+// In all these, 'low' means it's definitely in the main primary map,
+// 'high' means it's definitely in the auxiliary table.
+
+static INLINE SecMap** get_secmap_low_ptr ( Addr a )
{
UWord pm_off =3D a >> 16;
# if VG_DEBUG_MEMORY >=3D 1
tl_assert(pm_off < N_PRIMARY_MAP);
# endif
- return primary_map[ pm_off ];
+ return &primary_map[ pm_off ];
}
=20
+static INLINE SecMap** get_secmap_high_ptr ( Addr a )
+{
+ AuxMapEnt* am =3D find_or_alloc_in_auxmap(a);
+ return &am->sm;
+}
+
+static SecMap** get_secmap_ptr ( Addr a )
+{
+ return ( a <=3D MAX_PRIMARY_ADDRESS=20
+ ? get_secmap_low_ptr(a)=20
+ : get_secmap_high_ptr(a));
+}
+
+static INLINE SecMap* get_secmap_readable_low ( Addr a )
+{
+ return *get_secmap_low_ptr(a);
+}
+
+static INLINE SecMap* get_secmap_readable_high ( Addr a )
+{
+ return *get_secmap_high_ptr(a);
+}
+
+static INLINE SecMap* get_secmap_writable_low(Addr a)
+{
+ SecMap** p =3D get_secmap_low_ptr(a);
+ if (EXPECTED_NOT_TAKEN(is_distinguished_sm(*p)))
+ *p =3D copy_for_writing(*p);
+ return *p;
+}
+
+static INLINE SecMap* get_secmap_writable_high ( Addr a )
+{
+ SecMap** p =3D get_secmap_high_ptr(a);
+ if (EXPECTED_NOT_TAKEN(is_distinguished_sm(*p)))
+ *p =3D copy_for_writing(*p);
+ return *p;
+}
+
/* Produce the secmap for 'a', either from the primary map or by
ensuring there is an entry for it in the aux primary map. The
secmap may be a distinguished one as the caller will only want to
@@ -398,14 +443,25 @@
*/
static SecMap* get_secmap_readable ( Addr a )
{
- if (a <=3D MAX_PRIMARY_ADDRESS) {
- return get_secmap_readable_low(a);
- } else {
- AuxMapEnt* am =3D find_or_alloc_in_auxmap(a);
- return am->sm;
- }
+ return ( a <=3D MAX_PRIMARY_ADDRESS
+ ? get_secmap_readable_low (a)
+ : get_secmap_readable_high(a) );
}
=20
+/* Produce the secmap for 'a', either from the primary map or by
+ ensuring there is an entry for it in the aux primary map. The
+ secmap may not be a distinguished one, since the caller will want
+ to be able to write it. If it is a distinguished secondary, make a
+ writable copy of it, install it, and return the copy instead. (COW
+ semantics).
+*/
+static SecMap* get_secmap_writable ( Addr a )
+{
+ return ( a <=3D MAX_PRIMARY_ADDRESS
+ ? get_secmap_writable_low (a)
+ : get_secmap_writable_high(a) );
+}
+
/* If 'a' has a SecMap, produce it. Else produce NULL. But don't
allocate one if one doesn't already exist. This is used by the
leak checker.
@@ -420,39 +476,6 @@
}
}
=20
-// Produce the secmap for 'a', where 'a' is known to be in the primary m=
ap.
-__attribute__((always_inline))
-static inline SecMap* get_secmap_writable_low(Addr a)
-{
- UWord pm_off =3D a >> 16;
-# if VG_DEBUG_MEMORY >=3D 1
- tl_assert(pm_off < N_PRIMARY_MAP);
-# endif
- if (EXPECTED_NOT_TAKEN(is_distinguished_sm(primary_map[pm_off])))
- primary_map[pm_off] =3D copy_for_writing(primary_map[pm_off]);
- return primary_map[pm_off];
-}
-
-/* Produce the secmap for 'a', either from the primary map or by
- ensuring there is an entry for it in the aux primary map. The
- secmap may not be a distinguished one, since the caller will want
- to be able to write it. If it is a distinguished secondary, make a
- writable copy of it, install it, and return the copy instead. (COW
- semantics).
-*/
-static SecMap* get_secmap_writable ( Addr a )
-{
- if (a <=3D MAX_PRIMARY_ADDRESS) {
- return get_secmap_writable_low(a);
- } else {
- AuxMapEnt* am =3D find_or_alloc_in_auxmap(a);
- if (is_distinguished_sm(am->sm))
- am->sm =3D copy_for_writing(am->sm);
- return am->sm;
- }
-}
-
-
/* --------------- Secondary V bit table ------------ */
=20
// Note: the nodes in this table can become stale. Eg. if you write a
@@ -559,7 +582,7 @@
=20
/* Returns the offset in memory of the byteno-th most significant byte
in a wordszB-sized word, given the specified endianness. */
-static inline UWord byte_offset_w ( UWord wordszB, Bool bigendian,=20
+static INLINE UWord byte_offset_w ( UWord wordszB, Bool bigendian,=20
UWord byteno ) {
return bigendian ? (wordszB-1-byteno) : byteno;
}
@@ -567,7 +590,7 @@
=20
/* --------------- Fundamental functions --------------- */
=20
-static inline
+static INLINE
void insert_vabits8_into_vabits32 ( Addr a, UChar vabits8, UChar* vabits=
32 )
{
UInt shift =3D (a & 3) << 1; // shift by 0, 2, 4, or 6
@@ -575,7 +598,7 @@
*vabits32 |=3D (vabits8 << shift); // mask in the two new bits
}
=20
-static inline
+static INLINE
void insert_vabits16_into_vabits32 ( Addr a, UChar vabits16, UChar* vabi=
ts32 )
{
UInt shift;
@@ -585,7 +608,7 @@
*vabits32 |=3D (vabits16 << shift); // mask in the four new bits
}
=20
-static inline
+static INLINE
UChar extract_vabits8_from_vabits32 ( Addr a, UChar vabits32 )
{
UInt shift =3D (a & 3) << 1; // shift by 0, 2, 4, or 6
@@ -593,7 +616,7 @@
return 0x3 & vabits32; // mask out the rest
}
=20
-static inline
+static INLINE
UChar extract_vabits16_from_vabits32 ( Addr a, UChar vabits32 )
{
UInt shift;
@@ -607,7 +630,7 @@
// clever things like combine the auxmap check (in
// get_secmap_{read,writ}able) with alignment checks.
=20
-static inline
+static INLINE
void set_vabits8 ( Addr a, UChar vabits8 )
{
SecMap* sm =3D get_secmap_writable(a);
@@ -615,7 +638,7 @@
insert_vabits8_into_vabits32( a, vabits8, &(sm->vabits32[sm_off]) );
}
=20
-static inline
+static INLINE
UChar get_vabits8 ( Addr a )
{
SecMap* sm =3D get_secmap_readable(a);
@@ -755,7 +778,7 @@
=20
//zz /* Reading/writing of the bitmaps, for aligned word-sized accesses.=
*/
//zz=20
-//zz static __inline__ UChar get_abits4_ALIGNED ( Addr a )
+//zz static INLINE UChar get_abits4_ALIGNED ( Addr a )
//zz {
//zz SecMap* sm;
//zz UInt sm_off;
@@ -772,7 +795,7 @@
//zz return abits8;
//zz }
//zz=20
-//zz static UInt __inline__ get_vbytes4_ALIGNED ( Addr a )
+//zz static UInt INLINE get_vbytes4_ALIGNED ( Addr a )
//zz {
//zz SecMap* sm =3D primary_map[PM_IDX(a)];
//zz UInt sm_off =3D SM_OFF(a);
@@ -784,7 +807,7 @@
//zz }
//zz=20
//zz=20
-//zz static void __inline__ set_vbytes4_ALIGNED ( Addr a, UInt vbytes )
+//zz static void INLINE set_vbytes4_ALIGNED ( Addr a, UInt vbytes )
//zz {
//zz SecMap* sm;
//zz UInt sm_off;
@@ -803,33 +826,6 @@
/*--- Setting permissions over address ranges. ---*/
/*------------------------------------------------------------*/
=20
-/* Given address 'a', find the place where the pointer to a's
- secondary map lives. If a falls into the primary map, the returned
- value points to one of the entries in primary_map[]. Otherwise,
- the auxiliary primary map is searched for 'a', or an entry is
- created for it; either way, the returned value points to the
- relevant AuxMapEnt's .sm field.
-
- The point of this is to enable set_address_range_perms to assign
- secondary maps in a uniform way, without worrying about whether a
- given secondary map is pointed to from the main or auxiliary
- primary map. =20
-*/
-
-static SecMap** find_secmap_binder_for_addr ( Addr a )
-{
- if (a > MAX_PRIMARY_ADDRESS) {
- AuxMapEnt* am =3D find_or_alloc_in_auxmap(a);
- return &am->sm;
- } else {
- UWord sec_no =3D (UWord)(a >> 16);
-# if VG_DEBUG_MEMORY >=3D 1
- tl_assert(sec_no < N_PRIMARY_MAP);
-# endif
- return &primary_map[sec_no];
- }
-}
-
static void set_address_range_perms ( Addr a, SizeT lenT, UWord vabits64=
,
UWord dsm_num )
{
@@ -837,7 +833,7 @@
SizeT lenA, lenB, len_to_next_secmap;
Addr aNext;
SecMap* sm;
- SecMap** binder;
+ SecMap** sm_ptr;
SecMap* example_dsm;
=20
PROF_EVENT(150, "set_address_range_perms");
@@ -936,19 +932,19 @@
//-------------------------------------------------------------------=
-----
=20
// If it's distinguished, make it undistinguished if necessary.
- binder =3D find_secmap_binder_for_addr(a);
- if (is_distinguished_sm(*binder)) {
- if (*binder =3D=3D example_dsm) {
+ sm_ptr =3D get_secmap_ptr(a);
+ if (is_distinguished_sm(*sm_ptr)) {
+ if (*sm_ptr =3D=3D example_dsm) {
// Sec-map already has the V+A bits that we want, so skip.
PROF_EVENT(154, "set_address_range_perms-dist-sm1-quick");
a =3D aNext;
lenA =3D 0;
} else {
PROF_EVENT(155, "set_address_range_perms-dist-sm1");
- *binder =3D copy_for_writing(*binder);
+ *sm_ptr =3D copy_for_writing(*sm_ptr);
}
}
- sm =3D *binder;
+ sm =3D *sm_ptr;
=20
// 1 byte steps
while (True) {
@@ -993,16 +989,16 @@
if (lenB < SM_SIZE) break;
tl_assert(is_start_of_sm(a));
PROF_EVENT(159, "set_address_range_perms-loop64K");
- binder =3D find_secmap_binder_for_addr(a);
- if (!is_distinguished_sm(*binder)) {
+ sm_ptr =3D get_secmap_ptr(a);
+ if (!is_distinguished_sm(*sm_ptr)) {
PROF_EVENT(160, "set_address_range_perms-loop64K-free-dist-sm")=
;
// Free the non-distinguished sec-map that we're replacing. Th=
is
// case happens moderately often, enough to be worthwhile.
- VG_(am_munmap_valgrind)((Addr)*binder, sizeof(SecMap));
+ VG_(am_munmap_valgrind)((Addr)*sm_ptr, sizeof(SecMap));
n_secmaps_deissued++; // Needed for the expensive sanity c=
heck
}
// Make the sec-map entry point to the example DSM
- *binder =3D example_dsm;
+ *sm_ptr =3D example_dsm;
lenB -=3D SM_SIZE;
a +=3D SM_SIZE;
}
@@ -1018,18 +1014,18 @@
tl_assert(is_start_of_sm(a) && lenB < SM_SIZE);
=20
// If it's distinguished, make it undistinguished if necessary.
- binder =3D find_secmap_binder_for_addr(a);
- if (is_distinguished_sm(*binder)) {
- if (*binder =3D=3D example_dsm) {
+ sm_ptr =3D get_secmap_ptr(a);
+ if (is_distinguished_sm(*sm_ptr)) {
+ if (*sm_ptr =3D=3D example_dsm) {
// Sec-map already has the V+A bits that we want, so stop.
PROF_EVENT(161, "set_address_range_perms-dist-sm2-quick");
return;
} else {
PROF_EVENT(162, "set_address_range_perms-dist-sm2");
- *binder =3D copy_for_writing(*binder);
+ *sm_ptr =3D copy_for_writing(*sm_ptr);
}
}
- sm =3D *binder;
+ sm =3D *sm_ptr;
=20
// 8-aligned, 8 byte steps
while (True) {
@@ -1107,7 +1103,7 @@
=20
/* --- Fast case permission setters, for dealing with stacks. --- */
=20
-static __inline__
+static INLINE
void make_aligned_word32_writable ( Addr a )
{
UWord sm_off;
@@ -1132,7 +1128,7 @@
}
=20
=20
-static __inline__
+static INLINE
void make_aligned_word32_noaccess ( Addr a )
{
UWord sm_off;
@@ -1158,7 +1154,7 @@
=20
=20
/* Nb: by "aligned" here we mean 8-byte aligned */
-static __inline__
+static INLINE
void make_aligned_word64_writable ( Addr a )
{
UWord sm_off64;
@@ -1183,7 +1179,7 @@
}
=20
=20
-static __inline__
+static INLINE
void make_aligned_word64_noaccess ( Addr a )
{
UWord sm_off64;
@@ -1438,32 +1434,25 @@
/* Idea is: go fast when
* 8-aligned and length is 128
* the sm is available in the main primary map
- * the address range falls entirely with a single
- secondary map
- * the SM is modifiable
- If all those conditions hold, just update the V+A bits
- by writing directly into the vabits array. =20
+ * the address range falls entirely with a single secondary map
+ If all those conditions hold, just update the V+A bits by writing
+ directly into the vabits array. (If the sm was distinguished, thi=
s
+ will make a copy and then write to it.)
*/
- if (EXPECTED_TAKEN( len =3D=3D 128
- && VG_IS_8_ALIGNED(base)=20
- )) {
+ if (EXPECTED_TAKEN( len =3D=3D 128 && VG_IS_8_ALIGNED(base) )) {
/* Now we know the address range is suitably sized and aligned. */
- UWord a_lo =3D (UWord)base;
- UWord a_hi =3D (UWord)(base + 127);
- UWord sec_lo =3D a_lo >> 16;
- UWord sec_hi =3D a_hi >> 16;
-
- if (EXPECTED_TAKEN( sec_lo =3D=3D sec_hi=20
- && sec_lo <=3D N_PRIMARY_MAP
- )) {
+ UWord a_lo =3D (UWord)(base);
+ UWord a_hi =3D (UWord)(base + 127);
+ tl_assert(a_lo < a_hi); // paranoia: detect overflow
+ if (a_hi < MAX_PRIMARY_ADDRESS) {
+ // Now we know the entire range is within the main primary map.
+ SecMap* sm =3D get_secmap_writable_low(a_lo);
+ SecMap* sm_hi =3D get_secmap_writable_low(a_hi);
/* Now we know that the entire address range falls within a
single secondary map, and that that secondary 'lives' in
the main primary map. */
- SecMap* sm =3D primary_map[sec_lo];
-
- if (EXPECTED_TAKEN( !is_distinguished_sm(sm) )) {
- /* And finally, now we know that the secondary in question
- is modifiable. */
+ if (EXPECTED_TAKEN(sm =3D=3D sm_hi)) {
+ // Finally, we know that the range is entirely within one se=
cmap.
UWord v_off =3D SM_OFF(a_lo);
UShort* p =3D (UShort*)(&sm->vabits32[v_off]);
p[ 0] =3D VA_BITS64_WRITABLE;
@@ -2646,7 +2635,7 @@
=20
/* ------------------------ Size =3D 8 ------------------------ */
=20
-static inline __attribute__((always_inline))
+static INLINE
ULong mc_LOADV8 ( Addr a, Bool isBigEndian )
{
UWord sm_off64, vabits64;
@@ -2692,7 +2681,7 @@
}
=20
=20
-static inline __attribute__((always_inline))
+static INLINE
void mc_STOREV8 ( Addr a, ULong vbytes, Bool isBigEndian )
{
UWord sm_off64, vabits64;
@@ -2754,7 +2743,7 @@
=20
/* ------------------------ Size =3D 4 ------------------------ */
=20
-static inline __attribute__((always_inline))
+static INLINE
UWord mc_LOADV4 ( Addr a, Bool isBigEndian )
{
UWord sm_off, vabits32;
@@ -2802,7 +2791,7 @@
}
=20
=20
-static inline __attribute__((always_inline))
+static INLINE
void mc_STOREV4 ( Addr a, UWord vbytes, Bool isBigEndian )
{
UWord sm_off, vabits32;
@@ -2896,7 +2885,7 @@
=20
/* ------------------------ Size =3D 2 ------------------------ */
=20
-static inline __attribute__((always_inline))
+static INLINE
UWord mc_LOADV2 ( Addr a, Bool isBigEndian )
{
UWord sm_off, vabits32;
@@ -2947,7 +2936,7 @@
}
=20
=20
-static inline __attribute__((always_inline))
+static INLINE
void mc_STOREV2 ( Addr a, UWord vbytes, Bool isBigEndian )
{
UWord sm_off, vabits32;
|