Charm++: Charm Source Code Documentation

00001 /* -*- c -*-
00002    ----------------------------------------------------------------
00003 
00004    Notice that the following BSD-style license applies to this one
00005    file (valgrind.h) only.  The rest of Valgrind is licensed under the
00006    terms of the GNU General Public License, version 2, unless
00007    otherwise indicated.  See the COPYING file in the source
00008    distribution for details.
00009 
00010    ----------------------------------------------------------------
00011 
00012    This file is part of Valgrind, a dynamic binary instrumentation
00013    framework.
00014 
00015    Copyright (C) 2000-2015 Julian Seward.  All rights reserved.
00016 
00017    Redistribution and use in source and binary forms, with or without
00018    modification, are permitted provided that the following conditions
00019    are met:
00020 
00021    1. Redistributions of source code must retain the above copyright
00022       notice, this list of conditions and the following disclaimer.
00023 
00024    2. The origin of this software must not be misrepresented; you must 
00025       not claim that you wrote the original software.  If you use this 
00026       software in a product, an acknowledgment in the product 
00027       documentation would be appreciated but is not required.
00028 
00029    3. Altered source versions must be plainly marked as such, and must
00030       not be misrepresented as being the original software.
00031 
00032    4. The name of the author may not be used to endorse or promote 
00033       products derived from this software without specific prior written 
00034       permission.
00035 
00036    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
00037    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
00038    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
00039    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
00040    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
00041    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
00042    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
00043    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
00044    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00045    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00046    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00047 
00048    ----------------------------------------------------------------
00049 
00050    Notice that the above BSD-style license applies to this one file
00051    (valgrind.h) only.  The entire rest of Valgrind is licensed under
00052    the terms of the GNU General Public License, version 2.  See the
00053    COPYING file in the source distribution for details.
00054 
00055    ---------------------------------------------------------------- 
00056 */
00057 
00058 
00059 /* This file is for inclusion into client (your!) code.
00060 
00061    You can use these macros to manipulate and query Valgrind's 
00062    execution inside your own programs.
00063 
00064    The resulting executables will still run without Valgrind, just a
00065    little bit more slowly than they otherwise would, but otherwise
00066    unchanged.  When not running on valgrind, each client request
00067    consumes very few (eg. 7) instructions, so the resulting performance
00068    loss is negligible unless you plan to execute client requests
00069    millions of times per second.  Nevertheless, if that is still a
00070    problem, you can compile with the NVALGRIND symbol defined (gcc
00071    -DNVALGRIND) so that client requests are not even compiled in.  */
00072 
00073 #ifndef __VALGRIND_H
00074 #define __VALGRIND_H
00075 
00076 
00077 /* ------------------------------------------------------------------ */
00078 /* VERSION NUMBER OF VALGRIND                                         */
00079 /* ------------------------------------------------------------------ */
00080 
00081 /* Specify Valgrind's version number, so that user code can
00082    conditionally compile based on our version number.  Note that these
00083    were introduced at version 3.6 and so do not exist in version 3.5
00084    or earlier.  The recommended way to use them to check for "version
00085    X.Y or later" is (eg)
00086 
00087 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
00088     && (__VALGRIND_MAJOR__ > 3                                   \
00089         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
00090 */
00091 #define __VALGRIND_MAJOR__    3
00092 #define __VALGRIND_MINOR__    11
00093 
00094 
00095 #include <stdarg.h>
00096 
00097 /* Nb: this file might be included in a file compiled with -ansi.  So
00098    we can't use C++ style "//" comments nor the "asm" keyword (instead
00099    use "__asm__"). */
00100 
00101 /* Derive some tags indicating what the target platform is.  Note
00102    that in this file we're using the compiler's CPP symbols for
00103    identifying architectures, which are different to the ones we use
00104    within the rest of Valgrind.  Note, __powerpc__ is active for both
00105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
00106    latter (on Linux, that is).
00107 
00108    Misc note: how to find out what's predefined in gcc by default:
00109    gcc -Wp,-dM somefile.c
00110 */
00111 #undef PLAT_x86_darwin
00112 #undef PLAT_amd64_darwin
00113 #undef PLAT_x86_win32
00114 #undef PLAT_amd64_win64
00115 #undef PLAT_x86_linux
00116 #undef PLAT_amd64_linux
00117 #undef PLAT_ppc32_linux
00118 #undef PLAT_ppc64be_linux
00119 #undef PLAT_ppc64le_linux
00120 #undef PLAT_arm_linux
00121 #undef PLAT_arm64_linux
00122 #undef PLAT_s390x_linux
00123 #undef PLAT_mips32_linux
00124 #undef PLAT_mips64_linux
00125 #undef PLAT_tilegx_linux
00126 #undef PLAT_x86_solaris
00127 #undef PLAT_amd64_solaris
00128 
00129 
00130 #if defined(__APPLE__) && defined(__i386__)
00131 #  define PLAT_x86_darwin 1
00132 #elif defined(__APPLE__) && defined(__x86_64__)
00133 #  define PLAT_amd64_darwin 1
00134 #elif (defined(__MINGW32__) && !defined(__MINGW64__)) \
00135       || defined(__CYGWIN32__) \
00136       || (defined(_WIN32) && defined(_M_IX86))
00137 #  define PLAT_x86_win32 1
00138 #elif defined(__MINGW64__) \
00139       || (defined(_WIN64) && defined(_M_X64))
00140 #  define PLAT_amd64_win64 1
00141 #elif defined(__linux__) && defined(__i386__)
00142 #  define PLAT_x86_linux 1
00143 #elif defined(__linux__) && defined(__x86_64__) && !defined(__ILP32__)
00144 #  define PLAT_amd64_linux 1
00145 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
00146 #  define PLAT_ppc32_linux 1
00147 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2
00148 /* Big Endian uses ELF version 1 */
00149 #  define PLAT_ppc64be_linux 1
00150 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2
00151 /* Little Endian uses ELF version 2 */
00152 #  define PLAT_ppc64le_linux 1
00153 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__)
00154 #  define PLAT_arm_linux 1
00155 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__)
00156 #  define PLAT_arm64_linux 1
00157 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
00158 #  define PLAT_s390x_linux 1
00159 #elif defined(__linux__) && defined(__mips__) && (__mips==64)
00160 #  define PLAT_mips64_linux 1
00161 #elif defined(__linux__) && defined(__mips__) && (__mips!=64)
00162 #  define PLAT_mips32_linux 1
00163 #elif defined(__linux__) && defined(__tilegx__)
00164 #  define PLAT_tilegx_linux 1
00165 #elif defined(__sun) && defined(__i386__)
00166 #  define PLAT_x86_solaris 1
00167 #elif defined(__sun) && defined(__x86_64__)
00168 #  define PLAT_amd64_solaris 1
00169 #else
00170 /* If we're not compiling for our target platform, don't generate
00171    any inline asms.  */
00172 #  if !defined(NVALGRIND)
00173 #    define NVALGRIND 1
00174 #  endif
00175 #endif
00176 
00177 
00178 /* ------------------------------------------------------------------ */
00179 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
00180 /* in here of use to end-users -- skip to the next section.           */
00181 /* ------------------------------------------------------------------ */
00182 
00183 /*
00184  * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
00185  * request. Accepts both pointers and integers as arguments.
00186  *
00187  * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind
00188  * client request that does not return a value.
00189 
00190  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
00191  * client request and whose value equals the client request result.  Accepts
00192  * both pointers and integers as arguments.  Note that such calls are not
00193  * necessarily pure functions -- they may have side effects.
00194  */
00195 
00196 #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default,            \
00197                                    _zzq_request, _zzq_arg1, _zzq_arg2,  \
00198                                    _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
00199   do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default),   \
00200                         (_zzq_request), (_zzq_arg1), (_zzq_arg2),       \
00201                         (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
00202 
00203 #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1,        \
00204                            _zzq_arg2,  _zzq_arg3, _zzq_arg4, _zzq_arg5) \
00205   do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                        \
00206                     (_zzq_request), (_zzq_arg1), (_zzq_arg2),           \
00207                     (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
00208 
00209 #if defined(NVALGRIND)
00210 
00211 /* Define NVALGRIND to completely remove the Valgrind magic sequence
00212    from the compiled code (analogous to NDEBUG's effects on
00213    assert()) */
00214 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00215         _zzq_default, _zzq_request,                               \
00216         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00217       (_zzq_default)
00218 
00219 #else  /* ! NVALGRIND */
00220 
00221 /* The following defines the magic code sequences which the JITter
00222    spots and handles magically.  Don't look too closely at them as
00223    they will rot your brain.
00224 
00225    The assembly code sequences for all architectures is in this one
00226    file.  This is because this file must be stand-alone, and we don't
00227    want to have multiple files.
00228 
00229    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
00230    value gets put in the return slot, so that everything works when
00231    this is executed not under Valgrind.  Args are passed in a memory
00232    block, and so there's no intrinsic limit to the number that could
00233    be passed, but it's currently five.
00234    
00235    The macro args are: 
00236       _zzq_rlval    result lvalue
00237       _zzq_default  default value (result returned when running on real CPU)
00238       _zzq_request  request code
00239       _zzq_arg1..5  request params
00240 
00241    The other two macros are used to support function wrapping, and are
00242    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
00243    guest's NRADDR pseudo-register and whatever other information is
00244    needed to safely run the call original from the wrapper: on
00245    ppc64-linux, the R2 value at the divert point is also needed.  This
00246    information is abstracted into a user-visible type, OrigFn.
00247 
00248    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
00249    guest, but guarantees that the branch instruction will not be
00250    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
00251    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
00252    complete inline asm, since it needs to be combined with more magic
00253    inline asm stuff to be useful.
00254 */
00255 
00256 /* ----------------- x86-{linux,darwin,solaris} ---------------- */
00257 
00258 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
00259     ||  (defined(PLAT_x86_win32) && defined(__GNUC__)) \
00260     ||  defined(PLAT_x86_solaris)
00261 
00262 typedef
00263    struct { 
00264       unsigned int nraddr; /* where's the code? */
00265    }
00266    OrigFn;
00267 
00268 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00269                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
00270                      "roll $29, %%edi ; roll $19, %%edi\n\t"
00271 
00272 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00273         _zzq_default, _zzq_request,                               \
00274         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00275   __extension__                                                   \
00276   ({volatile unsigned int _zzq_args[6];                           \
00277     volatile unsigned int _zzq_result;                            \
00278     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
00279     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
00280     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
00281     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
00282     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
00283     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
00284     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00285                      /* %EDX = client_request ( %EAX ) */         \
00286                      "xchgl %%ebx,%%ebx"                          \
00287                      : "=d" (_zzq_result)                         \
00288                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
00289                      : "cc", "memory"                             \
00290                     );                                            \
00291     _zzq_result;                                                  \
00292   })
00293 
00294 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00295   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00296     volatile unsigned int __addr;                                 \
00297     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00298                      /* %EAX = guest_NRADDR */                    \
00299                      "xchgl %%ecx,%%ecx"                          \
00300                      : "=a" (__addr)                              \
00301                      :                                            \
00302                      : "cc", "memory"                             \
00303                     );                                            \
00304     _zzq_orig->nraddr = __addr;                                   \
00305   }
00306 
00307 #define VALGRIND_CALL_NOREDIR_EAX                                 \
00308                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00309                      /* call-noredir *%EAX */                     \
00310                      "xchgl %%edx,%%edx\n\t"
00311 
00312 #define VALGRIND_VEX_INJECT_IR()                                 \
00313  do {                                                            \
00314     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00315                      "xchgl %%edi,%%edi\n\t"                     \
00316                      : : : "cc", "memory"                        \
00317                     );                                           \
00318  } while (0)
00319 
00320 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__)
00321           || PLAT_x86_solaris */
00322 
00323 /* ------------------------- x86-Win32 ------------------------- */
00324 
00325 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
00326 
00327 typedef
00328    struct { 
00329       unsigned int nraddr; /* where's the code? */
00330    }
00331    OrigFn;
00332 
00333 #if defined(_MSC_VER)
00334 
00335 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00336                      __asm rol edi, 3  __asm rol edi, 13          \
00337                      __asm rol edi, 29 __asm rol edi, 19
00338 
00339 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00340         _zzq_default, _zzq_request,                               \
00341         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00342     valgrind_do_client_request_expr((uintptr_t)(_zzq_default),    \
00343         (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1),        \
00344         (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3),           \
00345         (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
00346 
00347 static __inline uintptr_t
00348 valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
00349                                 uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
00350                                 uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
00351                                 uintptr_t _zzq_arg5)
00352 {
00353     volatile uintptr_t _zzq_args[6];
00354     volatile unsigned int _zzq_result;
00355     _zzq_args[0] = (uintptr_t)(_zzq_request);
00356     _zzq_args[1] = (uintptr_t)(_zzq_arg1);
00357     _zzq_args[2] = (uintptr_t)(_zzq_arg2);
00358     _zzq_args[3] = (uintptr_t)(_zzq_arg3);
00359     _zzq_args[4] = (uintptr_t)(_zzq_arg4);
00360     _zzq_args[5] = (uintptr_t)(_zzq_arg5);
00361     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
00362             __SPECIAL_INSTRUCTION_PREAMBLE
00363             /* %EDX = client_request ( %EAX ) */
00364             __asm xchg ebx,ebx
00365             __asm mov _zzq_result, edx
00366     }
00367     return _zzq_result;
00368 }
00369 
00370 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00371   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00372     volatile unsigned int __addr;                                 \
00373     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
00374             /* %EAX = guest_NRADDR */                             \
00375             __asm xchg ecx,ecx                                    \
00376             __asm mov __addr, eax                                 \
00377     }                                                             \
00378     _zzq_orig->nraddr = __addr;                                   \
00379   }
00380 
00381 #define VALGRIND_CALL_NOREDIR_EAX ERROR
00382 
00383 #define VALGRIND_VEX_INJECT_IR()                                 \
00384  do {                                                            \
00385     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                       \
00386             __asm xchg edi,edi                                   \
00387     }                                                            \
00388  } while (0)
00389 
00390 #else
00391 #error Unsupported compiler.
00392 #endif
00393 
00394 #endif /* PLAT_x86_win32 */
00395 
00396 /* ----------------- amd64-{linux,darwin,solaris} --------------- */
00397 
00398 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
00399     ||  defined(PLAT_amd64_solaris) \
00400     ||  (defined(PLAT_amd64_win64) && defined(__GNUC__))
00401 
00402 typedef
00403    struct { 
00404       unsigned long int nraddr; /* where's the code? */
00405    }
00406    OrigFn;
00407 
00408 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00409                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
00410                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
00411 
00412 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00413         _zzq_default, _zzq_request,                               \
00414         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00415     __extension__                                                 \
00416     ({ volatile unsigned long int _zzq_args[6];                   \
00417     volatile unsigned long int _zzq_result;                       \
00418     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
00419     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
00420     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
00421     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
00422     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
00423     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
00424     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00425                      /* %RDX = client_request ( %RAX ) */         \
00426                      "xchgq %%rbx,%%rbx"                          \
00427                      : "=d" (_zzq_result)                         \
00428                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
00429                      : "cc", "memory"                             \
00430                     );                                            \
00431     _zzq_result;                                                  \
00432     })
00433 
00434 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00435   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00436     volatile unsigned long int __addr;                            \
00437     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00438                      /* %RAX = guest_NRADDR */                    \
00439                      "xchgq %%rcx,%%rcx"                          \
00440                      : "=a" (__addr)                              \
00441                      :                                            \
00442                      : "cc", "memory"                             \
00443                     );                                            \
00444     _zzq_orig->nraddr = __addr;                                   \
00445   }
00446 
00447 #define VALGRIND_CALL_NOREDIR_RAX                                 \
00448                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00449                      /* call-noredir *%RAX */                     \
00450                      "xchgq %%rdx,%%rdx\n\t"
00451 
00452 #define VALGRIND_VEX_INJECT_IR()                                 \
00453  do {                                                            \
00454     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00455                      "xchgq %%rdi,%%rdi\n\t"                     \
00456                      : : : "cc", "memory"                        \
00457                     );                                           \
00458  } while (0)
00459 
00460 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */
00461 
00462 /* ------------------------- amd64-Win64 ------------------------- */
00463 
00464 #if defined(PLAT_amd64_win64) && !defined(__GNUC__)
00465 
00466 #error Unsupported compiler.
00467 
00468 #endif /* PLAT_amd64_win64 */
00469 
00470 /* ------------------------ ppc32-linux ------------------------ */
00471 
00472 #if defined(PLAT_ppc32_linux)
00473 
00474 typedef
00475    struct { 
00476       unsigned int nraddr; /* where's the code? */
00477    }
00478    OrigFn;
00479 
00480 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00481                     "rlwinm 0,0,3,0,31  ; rlwinm 0,0,13,0,31\n\t" \
00482                     "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t"
00483 
00484 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00485         _zzq_default, _zzq_request,                               \
00486         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00487                                                                   \
00488     __extension__                                                 \
00489   ({         unsigned int  _zzq_args[6];                          \
00490              unsigned int  _zzq_result;                           \
00491              unsigned int* _zzq_ptr;                              \
00492     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
00493     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
00494     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
00495     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
00496     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
00497     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
00498     _zzq_ptr = _zzq_args;                                         \
00499     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
00500                      "mr 4,%2\n\t" /*ptr*/                        \
00501                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00502                      /* %R3 = client_request ( %R4 ) */           \
00503                      "or 1,1,1\n\t"                               \
00504                      "mr %0,3"     /*result*/                     \
00505                      : "=b" (_zzq_result)                         \
00506                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
00507                      : "cc", "memory", "r3", "r4");               \
00508     _zzq_result;                                                  \
00509     })
00510 
00511 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00512   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00513     unsigned int __addr;                                          \
00514     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00515                      /* %R3 = guest_NRADDR */                     \
00516                      "or 2,2,2\n\t"                               \
00517                      "mr %0,3"                                    \
00518                      : "=b" (__addr)                              \
00519                      :                                            \
00520                      : "cc", "memory", "r3"                       \
00521                     );                                            \
00522     _zzq_orig->nraddr = __addr;                                   \
00523   }
00524 
00525 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
00526                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00527                      /* branch-and-link-to-noredir *%R11 */       \
00528                      "or 3,3,3\n\t"
00529 
00530 #define VALGRIND_VEX_INJECT_IR()                                 \
00531  do {                                                            \
00532     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00533                      "or 5,5,5\n\t"                              \
00534                     );                                           \
00535  } while (0)
00536 
00537 #endif /* PLAT_ppc32_linux */
00538 
00539 /* ------------------------ ppc64-linux ------------------------ */
00540 
00541 #if defined(PLAT_ppc64be_linux)
00542 
00543 typedef
00544    struct { 
00545       unsigned long int nraddr; /* where's the code? */
00546       unsigned long int r2;  /* what tocptr do we need? */
00547    }
00548    OrigFn;
00549 
00550 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00551                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
00552                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
00553 
00554 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00555         _zzq_default, _zzq_request,                               \
00556         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00557                                                                   \
00558   __extension__                                                   \
00559   ({         unsigned long int  _zzq_args[6];                     \
00560              unsigned long int  _zzq_result;                      \
00561              unsigned long int* _zzq_ptr;                         \
00562     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
00563     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
00564     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
00565     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
00566     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
00567     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
00568     _zzq_ptr = _zzq_args;                                         \
00569     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
00570                      "mr 4,%2\n\t" /*ptr*/                        \
00571                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00572                      /* %R3 = client_request ( %R4 ) */           \
00573                      "or 1,1,1\n\t"                               \
00574                      "mr %0,3"     /*result*/                     \
00575                      : "=b" (_zzq_result)                         \
00576                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
00577                      : "cc", "memory", "r3", "r4");               \
00578     _zzq_result;                                                  \
00579   })
00580 
00581 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00582   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00583     unsigned long int __addr;                                     \
00584     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00585                      /* %R3 = guest_NRADDR */                     \
00586                      "or 2,2,2\n\t"                               \
00587                      "mr %0,3"                                    \
00588                      : "=b" (__addr)                              \
00589                      :                                            \
00590                      : "cc", "memory", "r3"                       \
00591                     );                                            \
00592     _zzq_orig->nraddr = __addr;                                   \
00593     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00594                      /* %R3 = guest_NRADDR_GPR2 */                \
00595                      "or 4,4,4\n\t"                               \
00596                      "mr %0,3"                                    \
00597                      : "=b" (__addr)                              \
00598                      :                                            \
00599                      : "cc", "memory", "r3"                       \
00600                     );                                            \
00601     _zzq_orig->r2 = __addr;                                       \
00602   }
00603 
00604 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
00605                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00606                      /* branch-and-link-to-noredir *%R11 */       \
00607                      "or 3,3,3\n\t"
00608 
00609 #define VALGRIND_VEX_INJECT_IR()                                 \
00610  do {                                                            \
00611     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00612                      "or 5,5,5\n\t"                              \
00613                     );                                           \
00614  } while (0)
00615 
00616 #endif /* PLAT_ppc64be_linux */
00617 
00618 #if defined(PLAT_ppc64le_linux)
00619 
00620 typedef
00621    struct {
00622       unsigned long int nraddr; /* where's the code? */
00623       unsigned long int r2;     /* what tocptr do we need? */
00624    }
00625    OrigFn;
00626 
00627 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00628                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
00629                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
00630 
00631 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00632         _zzq_default, _zzq_request,                               \
00633         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00634                                                                   \
00635   __extension__                                                   \
00636   ({         unsigned long int  _zzq_args[6];                     \
00637              unsigned long int  _zzq_result;                      \
00638              unsigned long int* _zzq_ptr;                         \
00639     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
00640     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
00641     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
00642     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
00643     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
00644     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
00645     _zzq_ptr = _zzq_args;                                         \
00646     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
00647                      "mr 4,%2\n\t" /*ptr*/                        \
00648                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00649                      /* %R3 = client_request ( %R4 ) */           \
00650                      "or 1,1,1\n\t"                               \
00651                      "mr %0,3"     /*result*/                     \
00652                      : "=b" (_zzq_result)                         \
00653                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
00654                      : "cc", "memory", "r3", "r4");               \
00655     _zzq_result;                                                  \
00656   })
00657 
00658 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00659   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00660     unsigned long int __addr;                                     \
00661     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00662                      /* %R3 = guest_NRADDR */                     \
00663                      "or 2,2,2\n\t"                               \
00664                      "mr %0,3"                                    \
00665                      : "=b" (__addr)                              \
00666                      :                                            \
00667                      : "cc", "memory", "r3"                       \
00668                     );                                            \
00669     _zzq_orig->nraddr = __addr;                                   \
00670     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00671                      /* %R3 = guest_NRADDR_GPR2 */                \
00672                      "or 4,4,4\n\t"                               \
00673                      "mr %0,3"                                    \
00674                      : "=b" (__addr)                              \
00675                      :                                            \
00676                      : "cc", "memory", "r3"                       \
00677                     );                                            \
00678     _zzq_orig->r2 = __addr;                                       \
00679   }
00680 
00681 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                   \
00682                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00683                      /* branch-and-link-to-noredir *%R12 */       \
00684                      "or 3,3,3\n\t"
00685 
00686 #define VALGRIND_VEX_INJECT_IR()                                 \
00687  do {                                                            \
00688     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00689                      "or 5,5,5\n\t"                              \
00690                     );                                           \
00691  } while (0)
00692 
00693 #endif /* PLAT_ppc64le_linux */
00694 
00695 /* ------------------------- arm-linux ------------------------- */
00696 
00697 #if defined(PLAT_arm_linux)
00698 
00699 typedef
00700    struct { 
00701       unsigned int nraddr; /* where's the code? */
00702    }
00703    OrigFn;
00704 
00705 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00706             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
00707             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
00708 
00709 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00710         _zzq_default, _zzq_request,                               \
00711         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00712                                                                   \
00713   __extension__                                                   \
00714   ({volatile unsigned int  _zzq_args[6];                          \
00715     volatile unsigned int  _zzq_result;                           \
00716     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
00717     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
00718     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
00719     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
00720     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
00721     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
00722     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
00723                      "mov r4, %2\n\t" /*ptr*/                     \
00724                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00725                      /* R3 = client_request ( R4 ) */             \
00726                      "orr r10, r10, r10\n\t"                      \
00727                      "mov %0, r3"     /*result*/                  \
00728                      : "=r" (_zzq_result)                         \
00729                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
00730                      : "cc","memory", "r3", "r4");                \
00731     _zzq_result;                                                  \
00732   })
00733 
00734 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00735   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00736     unsigned int __addr;                                          \
00737     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00738                      /* R3 = guest_NRADDR */                      \
00739                      "orr r11, r11, r11\n\t"                      \
00740                      "mov %0, r3"                                 \
00741                      : "=r" (__addr)                              \
00742                      :                                            \
00743                      : "cc", "memory", "r3"                       \
00744                     );                                            \
00745     _zzq_orig->nraddr = __addr;                                   \
00746   }
00747 
00748 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
00749                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00750                      /* branch-and-link-to-noredir *%R4 */        \
00751                      "orr r12, r12, r12\n\t"
00752 
00753 #define VALGRIND_VEX_INJECT_IR()                                 \
00754  do {                                                            \
00755     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00756                      "orr r9, r9, r9\n\t"                        \
00757                      : : : "cc", "memory"                        \
00758                     );                                           \
00759  } while (0)
00760 
00761 #endif /* PLAT_arm_linux */
00762 
00763 /* ------------------------ arm64-linux ------------------------- */
00764 
00765 #if defined(PLAT_arm64_linux)
00766 
00767 typedef
00768    struct { 
00769       unsigned long int nraddr; /* where's the code? */
00770    }
00771    OrigFn;
00772 
00773 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
00774             "ror x12, x12, #3  ;  ror x12, x12, #13 \n\t"         \
00775             "ror x12, x12, #51 ;  ror x12, x12, #61 \n\t"
00776 
00777 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00778         _zzq_default, _zzq_request,                               \
00779         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00780                                                                   \
00781   __extension__                                                   \
00782   ({volatile unsigned long int  _zzq_args[6];                     \
00783     volatile unsigned long int  _zzq_result;                      \
00784     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
00785     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
00786     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
00787     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
00788     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
00789     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
00790     __asm__ volatile("mov x3, %1\n\t" /*default*/                 \
00791                      "mov x4, %2\n\t" /*ptr*/                     \
00792                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00793                      /* X3 = client_request ( X4 ) */             \
00794                      "orr x10, x10, x10\n\t"                      \
00795                      "mov %0, x3"     /*result*/                  \
00796                      : "=r" (_zzq_result)                         \
00797                      : "r" ((unsigned long int)(_zzq_default)),   \
00798                        "r" (&_zzq_args[0])                        \
00799                      : "cc","memory", "x3", "x4");                \
00800     _zzq_result;                                                  \
00801   })
00802 
00803 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00804   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00805     unsigned long int __addr;                                     \
00806     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00807                      /* X3 = guest_NRADDR */                      \
00808                      "orr x11, x11, x11\n\t"                      \
00809                      "mov %0, x3"                                 \
00810                      : "=r" (__addr)                              \
00811                      :                                            \
00812                      : "cc", "memory", "x3"                       \
00813                     );                                            \
00814     _zzq_orig->nraddr = __addr;                                   \
00815   }
00816 
00817 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                    \
00818                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00819                      /* branch-and-link-to-noredir X8 */          \
00820                      "orr x12, x12, x12\n\t"
00821 
00822 #define VALGRIND_VEX_INJECT_IR()                                 \
00823  do {                                                            \
00824     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00825                      "orr x9, x9, x9\n\t"                        \
00826                      : : : "cc", "memory"                        \
00827                     );                                           \
00828  } while (0)
00829 
00830 #endif /* PLAT_arm64_linux */
00831 
00832 /* ------------------------ s390x-linux ------------------------ */
00833 
00834 #if defined(PLAT_s390x_linux)
00835 
00836 typedef
00837   struct {
00838      unsigned long int nraddr; /* where's the code? */
00839   }
00840   OrigFn;
00841 
00842 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
00843  * code. This detection is implemented in platform specific toIR.c
00844  * (e.g. VEX/priv/guest_s390_decoder.c).
00845  */
00846 #define __SPECIAL_INSTRUCTION_PREAMBLE                           \
00847                      "lr 15,15\n\t"                              \
00848                      "lr 1,1\n\t"                                \
00849                      "lr 2,2\n\t"                                \
00850                      "lr 3,3\n\t"
00851 
00852 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
00853 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
00854 #define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
00855 #define __VEX_INJECT_IR_CODE  "lr 5,5\n\t"
00856 
00857 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                         \
00858        _zzq_default, _zzq_request,                               \
00859        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
00860   __extension__                                                  \
00861  ({volatile unsigned long int _zzq_args[6];                      \
00862    volatile unsigned long int _zzq_result;                       \
00863    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
00864    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
00865    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
00866    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
00867    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
00868    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
00869    __asm__ volatile(/* r2 = args */                              \
00870                     "lgr 2,%1\n\t"                               \
00871                     /* r3 = default */                           \
00872                     "lgr 3,%2\n\t"                               \
00873                     __SPECIAL_INSTRUCTION_PREAMBLE               \
00874                     __CLIENT_REQUEST_CODE                        \
00875                     /* results = r3 */                           \
00876                     "lgr %0, 3\n\t"                              \
00877                     : "=d" (_zzq_result)                         \
00878                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
00879                     : "cc", "2", "3", "memory"                   \
00880                    );                                            \
00881    _zzq_result;                                                  \
00882  })
00883 
00884 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
00885  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00886    volatile unsigned long int __addr;                            \
00887    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00888                     __GET_NR_CONTEXT_CODE                        \
00889                     "lgr %0, 3\n\t"                              \
00890                     : "=a" (__addr)                              \
00891                     :                                            \
00892                     : "cc", "3", "memory"                        \
00893                    );                                            \
00894    _zzq_orig->nraddr = __addr;                                   \
00895  }
00896 
00897 #define VALGRIND_CALL_NOREDIR_R1                                 \
00898                     __SPECIAL_INSTRUCTION_PREAMBLE               \
00899                     __CALL_NO_REDIR_CODE
00900 
00901 #define VALGRIND_VEX_INJECT_IR()                                 \
00902  do {                                                            \
00903     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00904                      __VEX_INJECT_IR_CODE);                      \
00905  } while (0)
00906 
00907 #endif /* PLAT_s390x_linux */
00908 
00909 /* ------------------------- mips32-linux ---------------- */
00910 
00911 #if defined(PLAT_mips32_linux)
00912 
00913 typedef
00914    struct { 
00915       unsigned int nraddr; /* where's the code? */
00916    }
00917    OrigFn;
00918 
00919 /* .word  0x342
00920  * .word  0x742
00921  * .word  0xC2
00922  * .word  0x4C2*/
00923 #define __SPECIAL_INSTRUCTION_PREAMBLE          \
00924                      "srl $0, $0, 13\n\t"       \
00925                      "srl $0, $0, 29\n\t"       \
00926                      "srl $0, $0, 3\n\t"        \
00927                      "srl $0, $0, 19\n\t"
00928                     
00929 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
00930        _zzq_default, _zzq_request,                                \
00931        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
00932   __extension__                                                   \
00933   ({ volatile unsigned int _zzq_args[6];                          \
00934     volatile unsigned int _zzq_result;                            \
00935     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
00936     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
00937     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
00938     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
00939     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
00940     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
00941         __asm__ volatile("move $11, %1\n\t" /*default*/           \
00942                      "move $12, %2\n\t" /*ptr*/                   \
00943                      __SPECIAL_INSTRUCTION_PREAMBLE               \
00944                      /* T3 = client_request ( T4 ) */             \
00945                      "or $13, $13, $13\n\t"                       \
00946                      "move %0, $11\n\t"     /*result*/            \
00947                      : "=r" (_zzq_result)                         \
00948                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
00949                      : "$11", "$12");                             \
00950     _zzq_result;                                                  \
00951   })
00952 
00953 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
00954   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
00955     volatile unsigned int __addr;                                 \
00956     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
00957                      /* %t9 = guest_NRADDR */                     \
00958                      "or $14, $14, $14\n\t"                       \
00959                      "move %0, $11"     /*result*/                \
00960                      : "=r" (__addr)                              \
00961                      :                                            \
00962                      : "$11"                                      \
00963                     );                                            \
00964     _zzq_orig->nraddr = __addr;                                   \
00965   }
00966 
00967 #define VALGRIND_CALL_NOREDIR_T9                                 \
00968                      __SPECIAL_INSTRUCTION_PREAMBLE              \
00969                      /* call-noredir *%t9 */                     \
00970                      "or $15, $15, $15\n\t"
00971 
00972 #define VALGRIND_VEX_INJECT_IR()                                 \
00973  do {                                                            \
00974     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
00975                      "or $11, $11, $11\n\t"                      \
00976                     );                                           \
00977  } while (0)
00978 
00979 
00980 #endif /* PLAT_mips32_linux */
00981 
00982 /* ------------------------- mips64-linux ---------------- */
00983 
00984 #if defined(PLAT_mips64_linux)
00985 
00986 typedef
00987    struct {
00988       unsigned long nraddr; /* where's the code? */
00989    }
00990    OrigFn;
00991 
00992 /* dsll $0,$0, 3
00993  * dsll $0,$0, 13
00994  * dsll $0,$0, 29
00995  * dsll $0,$0, 19*/
00996 #define __SPECIAL_INSTRUCTION_PREAMBLE                              \
00997                      "dsll $0,$0, 3 ; dsll $0,$0,13\n\t"            \
00998                      "dsll $0,$0,29 ; dsll $0,$0,19\n\t"
00999 
01000 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                            \
01001        _zzq_default, _zzq_request,                                  \
01002        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)       \
01003   __extension__                                                     \
01004   ({ volatile unsigned long int _zzq_args[6];                       \
01005     volatile unsigned long int _zzq_result;                         \
01006     _zzq_args[0] = (unsigned long int)(_zzq_request);               \
01007     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                  \
01008     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                  \
01009     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                  \
01010     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                  \
01011     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                  \
01012         __asm__ volatile("move $11, %1\n\t" /*default*/             \
01013                          "move $12, %2\n\t" /*ptr*/                 \
01014                          __SPECIAL_INSTRUCTION_PREAMBLE             \
01015                          /* $11 = client_request ( $12 ) */         \
01016                          "or $13, $13, $13\n\t"                     \
01017                          "move %0, $11\n\t"     /*result*/          \
01018                          : "=r" (_zzq_result)                       \
01019                          : "r" (_zzq_default), "r" (&_zzq_args[0])  \
01020                          : "$11", "$12");                           \
01021     _zzq_result;                                                    \
01022   })
01023 
01024 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                         \
01025   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                     \
01026     volatile unsigned long int __addr;                              \
01027     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
01028                      /* $11 = guest_NRADDR */                       \
01029                      "or $14, $14, $14\n\t"                         \
01030                      "move %0, $11"     /*result*/                  \
01031                      : "=r" (__addr)                                \
01032                      :                                              \
01033                      : "$11");                                      \
01034     _zzq_orig->nraddr = __addr;                                     \
01035   }
01036 
01037 #define VALGRIND_CALL_NOREDIR_T9                                    \
01038                      __SPECIAL_INSTRUCTION_PREAMBLE                 \
01039                      /* call-noredir $25 */                         \
01040                      "or $15, $15, $15\n\t"
01041 
01042 #define VALGRIND_VEX_INJECT_IR()                                    \
01043  do {                                                               \
01044     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
01045                      "or $11, $11, $11\n\t"                         \
01046                     );                                              \
01047  } while (0)
01048 
01049 #endif /* PLAT_mips64_linux */
01050 
01051 /* ------------------------ tilegx-linux --------------- */
01052 #if defined(PLAT_tilegx_linux)
01053 
01054 typedef
01055    struct {
01056       unsigned long long int nraddr; /* where's the code? */
01057    }
01058    OrigFn;
01059 /*** special instruction sequence.
01060      0:02b3c7ff91234fff { moveli zero, 4660 ; moveli zero, 22136 }
01061      8:0091a7ff95678fff { moveli zero, 22136 ; moveli zero, 4660 }
01062 ****/
01063 
01064 #define __SPECIAL_INSTRUCTION_PREAMBLE                             \
01065    ".quad  0x02b3c7ff91234fff\n"                                   \
01066    ".quad  0x0091a7ff95678fff\n"
01067 
01068 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                           \
01069    _zzq_default, _zzq_request,                                      \
01070    _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)          \
01071    ({ volatile unsigned long long int _zzq_args[6];                \
01072       volatile unsigned long long int _zzq_result;                 \
01073       _zzq_args[0] = (unsigned long long int)(_zzq_request);       \
01074       _zzq_args[1] = (unsigned long long int)(_zzq_arg1);          \
01075       _zzq_args[2] = (unsigned long long int)(_zzq_arg2);          \
01076       _zzq_args[3] = (unsigned long long int)(_zzq_arg3);          \
01077       _zzq_args[4] = (unsigned long long int)(_zzq_arg4);          \
01078       _zzq_args[5] = (unsigned long long int)(_zzq_arg5);          \
01079       __asm__ volatile("move r11, %1\n\t" /*default*/              \
01080                        "move r12, %2\n\t" /*ptr*/                  \
01081                        __SPECIAL_INSTRUCTION_PREAMBLE              \
01082                        /* r11 = client_request */                  \
01083                        "or r13, r13, r13\n\t"                      \
01084                        "move %0, r11\n\t"     /*result*/           \
01085                        : "=r" (_zzq_result)                        \
01086                        : "r" (_zzq_default), "r" (&_zzq_args[0])   \
01087                        : "memory", "r11", "r12");                  \
01088       _zzq_result;                                                 \
01089    })
01090 
01091 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                        \
01092    {  volatile OrigFn* _zzq_orig = &(_zzq_rlval);                  \
01093       volatile unsigned long long int __addr;                      \
01094       __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
01095                        /* r11 = guest_NRADDR */                    \
01096                        "or r14, r14, r14\n"                        \
01097                        "move %0, r11\n"                            \
01098                        : "=r" (__addr)                             \
01099                        :                                           \
01100                        : "memory", "r11"                           \
01101                        );                                          \
01102       _zzq_orig->nraddr = __addr;                                  \
01103    }
01104 
01105 #define VALGRIND_CALL_NOREDIR_R12                                  \
01106    __SPECIAL_INSTRUCTION_PREAMBLE                                  \
01107    "or r15, r15, r15\n\t"
01108 
01109 #define VALGRIND_VEX_INJECT_IR()                                   \
01110    do {                                                            \
01111       __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
01112                        "or r11, r11, r11\n\t"                      \
01113                        );                                          \
01114    } while (0)
01115 
01116 #endif /* PLAT_tilegx_linux */
01117 
01118 /* Insert assembly code for other platforms here... */
01119 
01120 #endif /* NVALGRIND */
01121 
01122 
01123 /* ------------------------------------------------------------------ */
01124 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
01125 /* ugly.  It's the least-worst tradeoff I can think of.               */
01126 /* ------------------------------------------------------------------ */
01127 
01128 /* This section defines magic (a.k.a appalling-hack) macros for doing
01129    guaranteed-no-redirection macros, so as to get from function
01130    wrappers to the functions they are wrapping.  The whole point is to
01131    construct standard call sequences, but to do the call itself with a
01132    special no-redirect call pseudo-instruction that the JIT
01133    understands and handles specially.  This section is long and
01134    repetitious, and I can't see a way to make it shorter.
01135 
01136    The naming scheme is as follows:
01137 
01138       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
01139 
01140    'W' stands for "word" and 'v' for "void".  Hence there are
01141    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
01142    and for each, the possibility of returning a word-typed result, or
01143    no result.
01144 */
01145 
01146 /* Use these to write the name of your wrapper.  NOTE: duplicates
01147    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h.  NOTE also: inserts
01148    the default behaviour equivalance class tag "0000" into the name.
01149    See pub_tool_redir.h for details -- normally you don't need to
01150    think about this, though. */
01151 
01152 /* Use an extra level of macroisation so as to ensure the soname/fnname
01153    args are fully macro-expanded before pasting them together. */
01154 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
01155 
01156 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
01157    VG_CONCAT4(_vgw00000ZU_,soname,_,fnname)
01158 
01159 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
01160    VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname)
01161 
01162 /* Use this macro from within a wrapper function to collect the
01163    context (address and possibly other info) of the original function.
01164    Once you have that you can then use it in one of the CALL_FN_
01165    macros.  The type of the argument _lval is OrigFn. */
01166 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
01167 
01168 /* Also provide end-user facilities for function replacement, rather
01169    than wrapping.  A replacement function differs from a wrapper in
01170    that it has no way to get hold of the original function being
01171    called, and hence no way to call onwards to it.  In a replacement
01172    function, VALGRIND_GET_ORIG_FN always returns zero. */
01173 
01174 #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname)                 \
01175    VG_CONCAT4(_vgr00000ZU_,soname,_,fnname)
01176 
01177 #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname)                 \
01178    VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname)
01179 
01180 /* Derivatives of the main macros below, for calling functions
01181    returning void. */
01182 
01183 #define CALL_FN_v_v(fnptr)                                        \
01184    do { volatile unsigned long _junk;                             \
01185         CALL_FN_W_v(_junk,fnptr); } while (0)
01186 
01187 #define CALL_FN_v_W(fnptr, arg1)                                  \
01188    do { volatile unsigned long _junk;                             \
01189         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
01190 
01191 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
01192    do { volatile unsigned long _junk;                             \
01193         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
01194 
01195 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
01196    do { volatile unsigned long _junk;                             \
01197         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
01198 
01199 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
01200    do { volatile unsigned long _junk;                             \
01201         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
01202 
01203 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
01204    do { volatile unsigned long _junk;                             \
01205         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
01206 
01207 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
01208    do { volatile unsigned long _junk;                             \
01209         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
01210 
01211 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
01212    do { volatile unsigned long _junk;                             \
01213         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
01214 
01215 /* ----------------- x86-{linux,darwin,solaris} ---------------- */
01216 
01217 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin) \
01218     ||  defined(PLAT_x86_solaris)
01219 
01220 /* These regs are trashed by the hidden call.  No need to mention eax
01221    as gcc can already see that, plus causes gcc to bomb. */
01222 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
01223 
01224 /* Macros to save and align the stack before making a function
01225    call and restore it afterwards as gcc may not keep the stack
01226    pointer aligned if it doesn't realise calls are being made
01227    to other functions. */
01228 
01229 #define VALGRIND_ALIGN_STACK               \
01230       "movl %%esp,%%edi\n\t"               \
01231       "andl $0xfffffff0,%%esp\n\t"
01232 #define VALGRIND_RESTORE_STACK             \
01233       "movl %%edi,%%esp\n\t"
01234 
01235 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
01236    long) == 4. */
01237 
01238 #define CALL_FN_W_v(lval, orig)                                   \
01239    do {                                                           \
01240       volatile OrigFn        _orig = (orig);                      \
01241       volatile unsigned long _argvec[1];                          \
01242       volatile unsigned long _res;                                \
01243       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01244       __asm__ volatile(                                           \
01245          VALGRIND_ALIGN_STACK                                     \
01246          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01247          VALGRIND_CALL_NOREDIR_EAX                                \
01248          VALGRIND_RESTORE_STACK                                   \
01249          : /*out*/   "=a" (_res)                                  \
01250          : /*in*/    "a" (&_argvec[0])                            \
01251          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01252       );                                                          \
01253       lval = (__typeof__(lval)) _res;                             \
01254    } while (0)
01255 
01256 #define CALL_FN_W_W(lval, orig, arg1)                             \
01257    do {                                                           \
01258       volatile OrigFn        _orig = (orig);                      \
01259       volatile unsigned long _argvec[2];                          \
01260       volatile unsigned long _res;                                \
01261       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01262       _argvec[1] = (unsigned long)(arg1);                         \
01263       __asm__ volatile(                                           \
01264          VALGRIND_ALIGN_STACK                                     \
01265          "subl $12, %%esp\n\t"                                    \
01266          "pushl 4(%%eax)\n\t"                                     \
01267          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01268          VALGRIND_CALL_NOREDIR_EAX                                \
01269          VALGRIND_RESTORE_STACK                                   \
01270          : /*out*/   "=a" (_res)                                  \
01271          : /*in*/    "a" (&_argvec[0])                            \
01272          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01273       );                                                          \
01274       lval = (__typeof__(lval)) _res;                             \
01275    } while (0)
01276 
01277 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
01278    do {                                                           \
01279       volatile OrigFn        _orig = (orig);                      \
01280       volatile unsigned long _argvec[3];                          \
01281       volatile unsigned long _res;                                \
01282       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01283       _argvec[1] = (unsigned long)(arg1);                         \
01284       _argvec[2] = (unsigned long)(arg2);                         \
01285       __asm__ volatile(                                           \
01286          VALGRIND_ALIGN_STACK                                     \
01287          "subl $8, %%esp\n\t"                                     \
01288          "pushl 8(%%eax)\n\t"                                     \
01289          "pushl 4(%%eax)\n\t"                                     \
01290          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01291          VALGRIND_CALL_NOREDIR_EAX                                \
01292          VALGRIND_RESTORE_STACK                                   \
01293          : /*out*/   "=a" (_res)                                  \
01294          : /*in*/    "a" (&_argvec[0])                            \
01295          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01296       );                                                          \
01297       lval = (__typeof__(lval)) _res;                             \
01298    } while (0)
01299 
01300 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
01301    do {                                                           \
01302       volatile OrigFn        _orig = (orig);                      \
01303       volatile unsigned long _argvec[4];                          \
01304       volatile unsigned long _res;                                \
01305       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01306       _argvec[1] = (unsigned long)(arg1);                         \
01307       _argvec[2] = (unsigned long)(arg2);                         \
01308       _argvec[3] = (unsigned long)(arg3);                         \
01309       __asm__ volatile(                                           \
01310          VALGRIND_ALIGN_STACK                                     \
01311          "subl $4, %%esp\n\t"                                     \
01312          "pushl 12(%%eax)\n\t"                                    \
01313          "pushl 8(%%eax)\n\t"                                     \
01314          "pushl 4(%%eax)\n\t"                                     \
01315          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01316          VALGRIND_CALL_NOREDIR_EAX                                \
01317          VALGRIND_RESTORE_STACK                                   \
01318          : /*out*/   "=a" (_res)                                  \
01319          : /*in*/    "a" (&_argvec[0])                            \
01320          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01321       );                                                          \
01322       lval = (__typeof__(lval)) _res;                             \
01323    } while (0)
01324 
01325 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
01326    do {                                                           \
01327       volatile OrigFn        _orig = (orig);                      \
01328       volatile unsigned long _argvec[5];                          \
01329       volatile unsigned long _res;                                \
01330       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01331       _argvec[1] = (unsigned long)(arg1);                         \
01332       _argvec[2] = (unsigned long)(arg2);                         \
01333       _argvec[3] = (unsigned long)(arg3);                         \
01334       _argvec[4] = (unsigned long)(arg4);                         \
01335       __asm__ volatile(                                           \
01336          VALGRIND_ALIGN_STACK                                     \
01337          "pushl 16(%%eax)\n\t"                                    \
01338          "pushl 12(%%eax)\n\t"                                    \
01339          "pushl 8(%%eax)\n\t"                                     \
01340          "pushl 4(%%eax)\n\t"                                     \
01341          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01342          VALGRIND_CALL_NOREDIR_EAX                                \
01343          VALGRIND_RESTORE_STACK                                   \
01344          : /*out*/   "=a" (_res)                                  \
01345          : /*in*/    "a" (&_argvec[0])                            \
01346          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01347       );                                                          \
01348       lval = (__typeof__(lval)) _res;                             \
01349    } while (0)
01350 
01351 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
01352    do {                                                           \
01353       volatile OrigFn        _orig = (orig);                      \
01354       volatile unsigned long _argvec[6];                          \
01355       volatile unsigned long _res;                                \
01356       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01357       _argvec[1] = (unsigned long)(arg1);                         \
01358       _argvec[2] = (unsigned long)(arg2);                         \
01359       _argvec[3] = (unsigned long)(arg3);                         \
01360       _argvec[4] = (unsigned long)(arg4);                         \
01361       _argvec[5] = (unsigned long)(arg5);                         \
01362       __asm__ volatile(                                           \
01363          VALGRIND_ALIGN_STACK                                     \
01364          "subl $12, %%esp\n\t"                                    \
01365          "pushl 20(%%eax)\n\t"                                    \
01366          "pushl 16(%%eax)\n\t"                                    \
01367          "pushl 12(%%eax)\n\t"                                    \
01368          "pushl 8(%%eax)\n\t"                                     \
01369          "pushl 4(%%eax)\n\t"                                     \
01370          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01371          VALGRIND_CALL_NOREDIR_EAX                                \
01372          VALGRIND_RESTORE_STACK                                   \
01373          : /*out*/   "=a" (_res)                                  \
01374          : /*in*/    "a" (&_argvec[0])                            \
01375          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01376       );                                                          \
01377       lval = (__typeof__(lval)) _res;                             \
01378    } while (0)
01379 
01380 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
01381    do {                                                           \
01382       volatile OrigFn        _orig = (orig);                      \
01383       volatile unsigned long _argvec[7];                          \
01384       volatile unsigned long _res;                                \
01385       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01386       _argvec[1] = (unsigned long)(arg1);                         \
01387       _argvec[2] = (unsigned long)(arg2);                         \
01388       _argvec[3] = (unsigned long)(arg3);                         \
01389       _argvec[4] = (unsigned long)(arg4);                         \
01390       _argvec[5] = (unsigned long)(arg5);                         \
01391       _argvec[6] = (unsigned long)(arg6);                         \
01392       __asm__ volatile(                                           \
01393          VALGRIND_ALIGN_STACK                                     \
01394          "subl $8, %%esp\n\t"                                     \
01395          "pushl 24(%%eax)\n\t"                                    \
01396          "pushl 20(%%eax)\n\t"                                    \
01397          "pushl 16(%%eax)\n\t"                                    \
01398          "pushl 12(%%eax)\n\t"                                    \
01399          "pushl 8(%%eax)\n\t"                                     \
01400          "pushl 4(%%eax)\n\t"                                     \
01401          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01402          VALGRIND_CALL_NOREDIR_EAX                                \
01403          VALGRIND_RESTORE_STACK                                   \
01404          : /*out*/   "=a" (_res)                                  \
01405          : /*in*/    "a" (&_argvec[0])                            \
01406          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01407       );                                                          \
01408       lval = (__typeof__(lval)) _res;                             \
01409    } while (0)
01410 
01411 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
01412                                  arg7)                            \
01413    do {                                                           \
01414       volatile OrigFn        _orig = (orig);                      \
01415       volatile unsigned long _argvec[8];                          \
01416       volatile unsigned long _res;                                \
01417       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01418       _argvec[1] = (unsigned long)(arg1);                         \
01419       _argvec[2] = (unsigned long)(arg2);                         \
01420       _argvec[3] = (unsigned long)(arg3);                         \
01421       _argvec[4] = (unsigned long)(arg4);                         \
01422       _argvec[5] = (unsigned long)(arg5);                         \
01423       _argvec[6] = (unsigned long)(arg6);                         \
01424       _argvec[7] = (unsigned long)(arg7);                         \
01425       __asm__ volatile(                                           \
01426          VALGRIND_ALIGN_STACK                                     \
01427          "subl $4, %%esp\n\t"                                     \
01428          "pushl 28(%%eax)\n\t"                                    \
01429          "pushl 24(%%eax)\n\t"                                    \
01430          "pushl 20(%%eax)\n\t"                                    \
01431          "pushl 16(%%eax)\n\t"                                    \
01432          "pushl 12(%%eax)\n\t"                                    \
01433          "pushl 8(%%eax)\n\t"                                     \
01434          "pushl 4(%%eax)\n\t"                                     \
01435          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01436          VALGRIND_CALL_NOREDIR_EAX                                \
01437          VALGRIND_RESTORE_STACK                                   \
01438          : /*out*/   "=a" (_res)                                  \
01439          : /*in*/    "a" (&_argvec[0])                            \
01440          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01441       );                                                          \
01442       lval = (__typeof__(lval)) _res;                             \
01443    } while (0)
01444 
01445 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
01446                                  arg7,arg8)                       \
01447    do {                                                           \
01448       volatile OrigFn        _orig = (orig);                      \
01449       volatile unsigned long _argvec[9];                          \
01450       volatile unsigned long _res;                                \
01451       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01452       _argvec[1] = (unsigned long)(arg1);                         \
01453       _argvec[2] = (unsigned long)(arg2);                         \
01454       _argvec[3] = (unsigned long)(arg3);                         \
01455       _argvec[4] = (unsigned long)(arg4);                         \
01456       _argvec[5] = (unsigned long)(arg5);                         \
01457       _argvec[6] = (unsigned long)(arg6);                         \
01458       _argvec[7] = (unsigned long)(arg7);                         \
01459       _argvec[8] = (unsigned long)(arg8);                         \
01460       __asm__ volatile(                                           \
01461          VALGRIND_ALIGN_STACK                                     \
01462          "pushl 32(%%eax)\n\t"                                    \
01463          "pushl 28(%%eax)\n\t"                                    \
01464          "pushl 24(%%eax)\n\t"                                    \
01465          "pushl 20(%%eax)\n\t"                                    \
01466          "pushl 16(%%eax)\n\t"                                    \
01467          "pushl 12(%%eax)\n\t"                                    \
01468          "pushl 8(%%eax)\n\t"                                     \
01469          "pushl 4(%%eax)\n\t"                                     \
01470          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01471          VALGRIND_CALL_NOREDIR_EAX                                \
01472          VALGRIND_RESTORE_STACK                                   \
01473          : /*out*/   "=a" (_res)                                  \
01474          : /*in*/    "a" (&_argvec[0])                            \
01475          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01476       );                                                          \
01477       lval = (__typeof__(lval)) _res;                             \
01478    } while (0)
01479 
01480 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
01481                                  arg7,arg8,arg9)                  \
01482    do {                                                           \
01483       volatile OrigFn        _orig = (orig);                      \
01484       volatile unsigned long _argvec[10];                         \
01485       volatile unsigned long _res;                                \
01486       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01487       _argvec[1] = (unsigned long)(arg1);                         \
01488       _argvec[2] = (unsigned long)(arg2);                         \
01489       _argvec[3] = (unsigned long)(arg3);                         \
01490       _argvec[4] = (unsigned long)(arg4);                         \
01491       _argvec[5] = (unsigned long)(arg5);                         \
01492       _argvec[6] = (unsigned long)(arg6);                         \
01493       _argvec[7] = (unsigned long)(arg7);                         \
01494       _argvec[8] = (unsigned long)(arg8);                         \
01495       _argvec[9] = (unsigned long)(arg9);                         \
01496       __asm__ volatile(                                           \
01497          VALGRIND_ALIGN_STACK                                     \
01498          "subl $12, %%esp\n\t"                                    \
01499          "pushl 36(%%eax)\n\t"                                    \
01500          "pushl 32(%%eax)\n\t"                                    \
01501          "pushl 28(%%eax)\n\t"                                    \
01502          "pushl 24(%%eax)\n\t"                                    \
01503          "pushl 20(%%eax)\n\t"                                    \
01504          "pushl 16(%%eax)\n\t"                                    \
01505          "pushl 12(%%eax)\n\t"                                    \
01506          "pushl 8(%%eax)\n\t"                                     \
01507          "pushl 4(%%eax)\n\t"                                     \
01508          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01509          VALGRIND_CALL_NOREDIR_EAX                                \
01510          VALGRIND_RESTORE_STACK                                   \
01511          : /*out*/   "=a" (_res)                                  \
01512          : /*in*/    "a" (&_argvec[0])                            \
01513          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01514       );                                                          \
01515       lval = (__typeof__(lval)) _res;                             \
01516    } while (0)
01517 
01518 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
01519                                   arg7,arg8,arg9,arg10)           \
01520    do {                                                           \
01521       volatile OrigFn        _orig = (orig);                      \
01522       volatile unsigned long _argvec[11];                         \
01523       volatile unsigned long _res;                                \
01524       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01525       _argvec[1] = (unsigned long)(arg1);                         \
01526       _argvec[2] = (unsigned long)(arg2);                         \
01527       _argvec[3] = (unsigned long)(arg3);                         \
01528       _argvec[4] = (unsigned long)(arg4);                         \
01529       _argvec[5] = (unsigned long)(arg5);                         \
01530       _argvec[6] = (unsigned long)(arg6);                         \
01531       _argvec[7] = (unsigned long)(arg7);                         \
01532       _argvec[8] = (unsigned long)(arg8);                         \
01533       _argvec[9] = (unsigned long)(arg9);                         \
01534       _argvec[10] = (unsigned long)(arg10);                       \
01535       __asm__ volatile(                                           \
01536          VALGRIND_ALIGN_STACK                                     \
01537          "subl $8, %%esp\n\t"                                     \
01538          "pushl 40(%%eax)\n\t"                                    \
01539          "pushl 36(%%eax)\n\t"                                    \
01540          "pushl 32(%%eax)\n\t"                                    \
01541          "pushl 28(%%eax)\n\t"                                    \
01542          "pushl 24(%%eax)\n\t"                                    \
01543          "pushl 20(%%eax)\n\t"                                    \
01544          "pushl 16(%%eax)\n\t"                                    \
01545          "pushl 12(%%eax)\n\t"                                    \
01546          "pushl 8(%%eax)\n\t"                                     \
01547          "pushl 4(%%eax)\n\t"                                     \
01548          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01549          VALGRIND_CALL_NOREDIR_EAX                                \
01550          VALGRIND_RESTORE_STACK                                   \
01551          : /*out*/   "=a" (_res)                                  \
01552          : /*in*/    "a" (&_argvec[0])                            \
01553          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01554       );                                                          \
01555       lval = (__typeof__(lval)) _res;                             \
01556    } while (0)
01557 
01558 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
01559                                   arg6,arg7,arg8,arg9,arg10,      \
01560                                   arg11)                          \
01561    do {                                                           \
01562       volatile OrigFn        _orig = (orig);                      \
01563       volatile unsigned long _argvec[12];                         \
01564       volatile unsigned long _res;                                \
01565       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01566       _argvec[1] = (unsigned long)(arg1);                         \
01567       _argvec[2] = (unsigned long)(arg2);                         \
01568       _argvec[3] = (unsigned long)(arg3);                         \
01569       _argvec[4] = (unsigned long)(arg4);                         \
01570       _argvec[5] = (unsigned long)(arg5);                         \
01571       _argvec[6] = (unsigned long)(arg6);                         \
01572       _argvec[7] = (unsigned long)(arg7);                         \
01573       _argvec[8] = (unsigned long)(arg8);                         \
01574       _argvec[9] = (unsigned long)(arg9);                         \
01575       _argvec[10] = (unsigned long)(arg10);                       \
01576       _argvec[11] = (unsigned long)(arg11);                       \
01577       __asm__ volatile(                                           \
01578          VALGRIND_ALIGN_STACK                                     \
01579          "subl $4, %%esp\n\t"                                     \
01580          "pushl 44(%%eax)\n\t"                                    \
01581          "pushl 40(%%eax)\n\t"                                    \
01582          "pushl 36(%%eax)\n\t"                                    \
01583          "pushl 32(%%eax)\n\t"                                    \
01584          "pushl 28(%%eax)\n\t"                                    \
01585          "pushl 24(%%eax)\n\t"                                    \
01586          "pushl 20(%%eax)\n\t"                                    \
01587          "pushl 16(%%eax)\n\t"                                    \
01588          "pushl 12(%%eax)\n\t"                                    \
01589          "pushl 8(%%eax)\n\t"                                     \
01590          "pushl 4(%%eax)\n\t"                                     \
01591          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01592          VALGRIND_CALL_NOREDIR_EAX                                \
01593          VALGRIND_RESTORE_STACK                                   \
01594          : /*out*/   "=a" (_res)                                  \
01595          : /*in*/    "a" (&_argvec[0])                            \
01596          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01597       );                                                          \
01598       lval = (__typeof__(lval)) _res;                             \
01599    } while (0)
01600 
01601 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
01602                                   arg6,arg7,arg8,arg9,arg10,      \
01603                                   arg11,arg12)                    \
01604    do {                                                           \
01605       volatile OrigFn        _orig = (orig);                      \
01606       volatile unsigned long _argvec[13];                         \
01607       volatile unsigned long _res;                                \
01608       _argvec[0] = (unsigned long)_orig.nraddr;                   \
01609       _argvec[1] = (unsigned long)(arg1);                         \
01610       _argvec[2] = (unsigned long)(arg2);                         \
01611       _argvec[3] = (unsigned long)(arg3);                         \
01612       _argvec[4] = (unsigned long)(arg4);                         \
01613       _argvec[5] = (unsigned long)(arg5);                         \
01614       _argvec[6] = (unsigned long)(arg6);                         \
01615       _argvec[7] = (unsigned long)(arg7);                         \
01616       _argvec[8] = (unsigned long)(arg8);                         \
01617       _argvec[9] = (unsigned long)(arg9);                         \
01618       _argvec[10] = (unsigned long)(arg10);                       \
01619       _argvec[11] = (unsigned long)(arg11);                       \
01620       _argvec[12] = (unsigned long)(arg12);                       \
01621       __asm__ volatile(                                           \
01622          VALGRIND_ALIGN_STACK                                     \
01623          "pushl 48(%%eax)\n\t"                                    \
01624          "pushl 44(%%eax)\n\t"                                    \
01625          "pushl 40(%%eax)\n\t"                                    \
01626          "pushl 36(%%eax)\n\t"                                    \
01627          "pushl 32(%%eax)\n\t"                                    \
01628          "pushl 28(%%eax)\n\t"                                    \
01629          "pushl 24(%%eax)\n\t"                                    \
01630          "pushl 20(%%eax)\n\t"                                    \
01631          "pushl 16(%%eax)\n\t"                                    \
01632          "pushl 12(%%eax)\n\t"                                    \
01633          "pushl 8(%%eax)\n\t"                                     \
01634          "pushl 4(%%eax)\n\t"                                     \
01635          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
01636          VALGRIND_CALL_NOREDIR_EAX                                \
01637          VALGRIND_RESTORE_STACK                                   \
01638          : /*out*/   "=a" (_res)                                  \
01639          : /*in*/    "a" (&_argvec[0])                            \
01640          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
01641       );                                                          \
01642       lval = (__typeof__(lval)) _res;                             \
01643    } while (0)
01644 
01645 #endif /* PLAT_x86_linux || PLAT_x86_darwin || PLAT_x86_solaris */
01646 
01647 /* ---------------- amd64-{linux,darwin,solaris} --------------- */
01648 
01649 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
01650     ||  defined(PLAT_amd64_solaris)
01651 
01652 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
01653 
01654 /* These regs are trashed by the hidden call. */
01655 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
01656                             "rdi", "r8", "r9", "r10", "r11"
01657 
01658 /* This is all pretty complex.  It's so as to make stack unwinding
01659    work reliably.  See bug 243270.  The basic problem is the sub and
01660    add of 128 of %rsp in all of the following macros.  If gcc believes
01661    the CFA is in %rsp, then unwinding may fail, because what's at the
01662    CFA is not what gcc "expected" when it constructs the CFIs for the
01663    places where the macros are instantiated.
01664 
01665    But we can't just add a CFI annotation to increase the CFA offset
01666    by 128, to match the sub of 128 from %rsp, because we don't know
01667    whether gcc has chosen %rsp as the CFA at that point, or whether it
01668    has chosen some other register (eg, %rbp).  In the latter case,
01669    adding a CFI annotation to change the CFA offset is simply wrong.
01670 
01671    So the solution is to get hold of the CFA using
01672    __builtin_dwarf_cfa(), put it in a known register, and add a
01673    CFI annotation to say what the register is.  We choose %rbp for
01674    this (perhaps perversely), because:
01675 
01676    (1) %rbp is already subject to unwinding.  If a new register was
01677        chosen then the unwinder would have to unwind it in all stack
01678        traces, which is expensive, and
01679 
01680    (2) %rbp is already subject to precise exception updates in the
01681        JIT.  If a new register was chosen, we'd have to have precise
01682        exceptions for it too, which reduces performance of the
01683        generated code.
01684 
01685    However .. one extra complication.  We can't just whack the result
01686    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
01687    list of trashed registers at the end of the inline assembly
01688    fragments; gcc won't allow %rbp to appear in that list.  Hence
01689    instead we need to stash %rbp in %r15 for the duration of the asm,
01690    and say that %r15 is trashed instead.  gcc seems happy to go with
01691    that.
01692 
01693    Oh .. and this all needs to be conditionalised so that it is
01694    unchanged from before this commit, when compiled with older gccs
01695    that don't support __builtin_dwarf_cfa.  Furthermore, since
01696    this header file is freestanding, it has to be independent of
01697    config.h, and so the following conditionalisation cannot depend on
01698    configure time checks.
01699 
01700    Although it's not clear from
01701    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
01702    this expression excludes Darwin.
01703    .cfi directives in Darwin assembly appear to be completely
01704    different and I haven't investigated how they work.
01705 
01706    For even more entertainment value, note we have to use the
01707    completely undocumented __builtin_dwarf_cfa(), which appears to
01708    really compute the CFA, whereas __builtin_frame_address(0) claims
01709    to but actually doesn't.  See
01710    https://bugs.kde.org/show_bug.cgi?id=243270#c47
01711 */
01712 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
01713 #  define __FRAME_POINTER                                         \
01714       ,"r"(__builtin_dwarf_cfa())
01715 #  define VALGRIND_CFI_PROLOGUE                                   \
01716       "movq %%rbp, %%r15\n\t"                                     \
01717       "movq %2, %%rbp\n\t"                                        \
01718       ".cfi_remember_state\n\t"                                   \
01719       ".cfi_def_cfa rbp, 0\n\t"
01720 #  define VALGRIND_CFI_EPILOGUE                                   \
01721       "movq %%r15, %%rbp\n\t"                                     \
01722       ".cfi_restore_state\n\t"
01723 #else
01724 #  define __FRAME_POINTER
01725 #  define VALGRIND_CFI_PROLOGUE
01726 #  define VALGRIND_CFI_EPILOGUE
01727 #endif
01728 
01729 /* Macros to save and align the stack before making a function
01730    call and restore it afterwards as gcc may not keep the stack
01731    pointer aligned if it doesn't realise calls are being made
01732    to other functions. */
01733 
01734 #define VALGRIND_ALIGN_STACK               \
01735       "movq %%rsp,%%r14\n\t"               \
01736       "andq $0xfffffffffffffff0,%%rsp\n\t"
01737 #define VALGRIND_RESTORE_STACK             \
01738       "movq %%r14,%%rsp\n\t"
01739 
01740 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
01741    long) == 8. */
01742 
01743 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
01744    macros.  In order not to trash the stack redzone, we need to drop
01745    %rsp by 128 before the hidden call, and restore afterwards.  The
01746    nastyness is that it is only by luck that the stack still appears
01747    to be unwindable during the hidden call - since then the behaviour
01748    of any routine using this macro does not match what the CFI data
01749    says.  Sigh.
01750 
01751    Why is this important?  Imagine that a wrapper has a stack
01752    allocated local, and passes to the hidden call, a pointer to it.
01753    Because gcc does not know about the hidden call, it may allocate
01754    that local in the redzone.  Unfortunately the hidden call may then
01755    trash it before it comes to use it.  So we must step clear of the
01756    redzone, for the duration of the hidden call, to make it safe.
01757 
01758    Probably the same problem afflicts the other redzone-style ABIs too
01759    (ppc64-linux); but for those, the stack is
01760    self describing (none of this CFI nonsense) so at least messing
01761    with the stack pointer doesn't give a danger of non-unwindable
01762    stack. */
01763 
01764 #define CALL_FN_W_v(lval, orig)                                        \
01765    do {                                                                \
01766       volatile OrigFn        _orig = (orig);                           \
01767       volatile unsigned long _argvec[1];                               \
01768       volatile unsigned long _res;                                     \
01769       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01770       __asm__ volatile(                                                \
01771          VALGRIND_CFI_PROLOGUE                                         \
01772          VALGRIND_ALIGN_STACK                                          \
01773          "subq $128,%%rsp\n\t"                                         \
01774          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01775          VALGRIND_CALL_NOREDIR_RAX                                     \
01776          VALGRIND_RESTORE_STACK                                        \
01777          VALGRIND_CFI_EPILOGUE                                         \
01778          : /*out*/   "=a" (_res)                                       \
01779          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01780          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01781       );                                                               \
01782       lval = (__typeof__(lval)) _res;                                  \
01783    } while (0)
01784 
01785 #define CALL_FN_W_W(lval, orig, arg1)                                  \
01786    do {                                                                \
01787       volatile OrigFn        _orig = (orig);                           \
01788       volatile unsigned long _argvec[2];                               \
01789       volatile unsigned long _res;                                     \
01790       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01791       _argvec[1] = (unsigned long)(arg1);                              \
01792       __asm__ volatile(                                                \
01793          VALGRIND_CFI_PROLOGUE                                         \
01794          VALGRIND_ALIGN_STACK                                          \
01795          "subq $128,%%rsp\n\t"                                         \
01796          "movq 8(%%rax), %%rdi\n\t"                                    \
01797          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01798          VALGRIND_CALL_NOREDIR_RAX                                     \
01799          VALGRIND_RESTORE_STACK                                        \
01800          VALGRIND_CFI_EPILOGUE                                         \
01801          : /*out*/   "=a" (_res)                                       \
01802          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01803          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01804       );                                                               \
01805       lval = (__typeof__(lval)) _res;                                  \
01806    } while (0)
01807 
01808 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                            \
01809    do {                                                                \
01810       volatile OrigFn        _orig = (orig);                           \
01811       volatile unsigned long _argvec[3];                               \
01812       volatile unsigned long _res;                                     \
01813       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01814       _argvec[1] = (unsigned long)(arg1);                              \
01815       _argvec[2] = (unsigned long)(arg2);                              \
01816       __asm__ volatile(                                                \
01817          VALGRIND_CFI_PROLOGUE                                         \
01818          VALGRIND_ALIGN_STACK                                          \
01819          "subq $128,%%rsp\n\t"                                         \
01820          "movq 16(%%rax), %%rsi\n\t"                                   \
01821          "movq 8(%%rax), %%rdi\n\t"                                    \
01822          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01823          VALGRIND_CALL_NOREDIR_RAX                                     \
01824          VALGRIND_RESTORE_STACK                                        \
01825          VALGRIND_CFI_EPILOGUE                                         \
01826          : /*out*/   "=a" (_res)                                       \
01827          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01828          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01829       );                                                               \
01830       lval = (__typeof__(lval)) _res;                                  \
01831    } while (0)
01832 
01833 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                      \
01834    do {                                                                \
01835       volatile OrigFn        _orig = (orig);                           \
01836       volatile unsigned long _argvec[4];                               \
01837       volatile unsigned long _res;                                     \
01838       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01839       _argvec[1] = (unsigned long)(arg1);                              \
01840       _argvec[2] = (unsigned long)(arg2);                              \
01841       _argvec[3] = (unsigned long)(arg3);                              \
01842       __asm__ volatile(                                                \
01843          VALGRIND_CFI_PROLOGUE                                         \
01844          VALGRIND_ALIGN_STACK                                          \
01845          "subq $128,%%rsp\n\t"                                         \
01846          "movq 24(%%rax), %%rdx\n\t"                                   \
01847          "movq 16(%%rax), %%rsi\n\t"                                   \
01848          "movq 8(%%rax), %%rdi\n\t"                                    \
01849          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01850          VALGRIND_CALL_NOREDIR_RAX                                     \
01851          VALGRIND_RESTORE_STACK                                        \
01852          VALGRIND_CFI_EPILOGUE                                         \
01853          : /*out*/   "=a" (_res)                                       \
01854          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01855          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01856       );                                                               \
01857       lval = (__typeof__(lval)) _res;                                  \
01858    } while (0)
01859 
01860 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)                \
01861    do {                                                                \
01862       volatile OrigFn        _orig = (orig);                           \
01863       volatile unsigned long _argvec[5];                               \
01864       volatile unsigned long _res;                                     \
01865       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01866       _argvec[1] = (unsigned long)(arg1);                              \
01867       _argvec[2] = (unsigned long)(arg2);                              \
01868       _argvec[3] = (unsigned long)(arg3);                              \
01869       _argvec[4] = (unsigned long)(arg4);                              \
01870       __asm__ volatile(                                                \
01871          VALGRIND_CFI_PROLOGUE                                         \
01872          VALGRIND_ALIGN_STACK                                          \
01873          "subq $128,%%rsp\n\t"                                         \
01874          "movq 32(%%rax), %%rcx\n\t"                                   \
01875          "movq 24(%%rax), %%rdx\n\t"                                   \
01876          "movq 16(%%rax), %%rsi\n\t"                                   \
01877          "movq 8(%%rax), %%rdi\n\t"                                    \
01878          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01879          VALGRIND_CALL_NOREDIR_RAX                                     \
01880          VALGRIND_RESTORE_STACK                                        \
01881          VALGRIND_CFI_EPILOGUE                                         \
01882          : /*out*/   "=a" (_res)                                       \
01883          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01884          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01885       );                                                               \
01886       lval = (__typeof__(lval)) _res;                                  \
01887    } while (0)
01888 
01889 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)             \
01890    do {                                                                \
01891       volatile OrigFn        _orig = (orig);                           \
01892       volatile unsigned long _argvec[6];                               \
01893       volatile unsigned long _res;                                     \
01894       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01895       _argvec[1] = (unsigned long)(arg1);                              \
01896       _argvec[2] = (unsigned long)(arg2);                              \
01897       _argvec[3] = (unsigned long)(arg3);                              \
01898       _argvec[4] = (unsigned long)(arg4);                              \
01899       _argvec[5] = (unsigned long)(arg5);                              \
01900       __asm__ volatile(                                                \
01901          VALGRIND_CFI_PROLOGUE                                         \
01902          VALGRIND_ALIGN_STACK                                          \
01903          "subq $128,%%rsp\n\t"                                         \
01904          "movq 40(%%rax), %%r8\n\t"                                    \
01905          "movq 32(%%rax), %%rcx\n\t"                                   \
01906          "movq 24(%%rax), %%rdx\n\t"                                   \
01907          "movq 16(%%rax), %%rsi\n\t"                                   \
01908          "movq 8(%%rax), %%rdi\n\t"                                    \
01909          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01910          VALGRIND_CALL_NOREDIR_RAX                                     \
01911          VALGRIND_RESTORE_STACK                                        \
01912          VALGRIND_CFI_EPILOGUE                                         \
01913          : /*out*/   "=a" (_res)                                       \
01914          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01915          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01916       );                                                               \
01917       lval = (__typeof__(lval)) _res;                                  \
01918    } while (0)
01919 
01920 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)        \
01921    do {                                                                \
01922       volatile OrigFn        _orig = (orig);                           \
01923       volatile unsigned long _argvec[7];                               \
01924       volatile unsigned long _res;                                     \
01925       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01926       _argvec[1] = (unsigned long)(arg1);                              \
01927       _argvec[2] = (unsigned long)(arg2);                              \
01928       _argvec[3] = (unsigned long)(arg3);                              \
01929       _argvec[4] = (unsigned long)(arg4);                              \
01930       _argvec[5] = (unsigned long)(arg5);                              \
01931       _argvec[6] = (unsigned long)(arg6);                              \
01932       __asm__ volatile(                                                \
01933          VALGRIND_CFI_PROLOGUE                                         \
01934          VALGRIND_ALIGN_STACK                                          \
01935          "subq $128,%%rsp\n\t"                                         \
01936          "movq 48(%%rax), %%r9\n\t"                                    \
01937          "movq 40(%%rax), %%r8\n\t"                                    \
01938          "movq 32(%%rax), %%rcx\n\t"                                   \
01939          "movq 24(%%rax), %%rdx\n\t"                                   \
01940          "movq 16(%%rax), %%rsi\n\t"                                   \
01941          "movq 8(%%rax), %%rdi\n\t"                                    \
01942          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01943          VALGRIND_CALL_NOREDIR_RAX                                     \
01944          VALGRIND_RESTORE_STACK                                        \
01945          VALGRIND_CFI_EPILOGUE                                         \
01946          : /*out*/   "=a" (_res)                                       \
01947          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01948          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01949       );                                                               \
01950       lval = (__typeof__(lval)) _res;                                  \
01951    } while (0)
01952 
01953 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
01954                                  arg7)                                 \
01955    do {                                                                \
01956       volatile OrigFn        _orig = (orig);                           \
01957       volatile unsigned long _argvec[8];                               \
01958       volatile unsigned long _res;                                     \
01959       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01960       _argvec[1] = (unsigned long)(arg1);                              \
01961       _argvec[2] = (unsigned long)(arg2);                              \
01962       _argvec[3] = (unsigned long)(arg3);                              \
01963       _argvec[4] = (unsigned long)(arg4);                              \
01964       _argvec[5] = (unsigned long)(arg5);                              \
01965       _argvec[6] = (unsigned long)(arg6);                              \
01966       _argvec[7] = (unsigned long)(arg7);                              \
01967       __asm__ volatile(                                                \
01968          VALGRIND_CFI_PROLOGUE                                         \
01969          VALGRIND_ALIGN_STACK                                          \
01970          "subq $136,%%rsp\n\t"                                         \
01971          "pushq 56(%%rax)\n\t"                                         \
01972          "movq 48(%%rax), %%r9\n\t"                                    \
01973          "movq 40(%%rax), %%r8\n\t"                                    \
01974          "movq 32(%%rax), %%rcx\n\t"                                   \
01975          "movq 24(%%rax), %%rdx\n\t"                                   \
01976          "movq 16(%%rax), %%rsi\n\t"                                   \
01977          "movq 8(%%rax), %%rdi\n\t"                                    \
01978          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
01979          VALGRIND_CALL_NOREDIR_RAX                                     \
01980          VALGRIND_RESTORE_STACK                                        \
01981          VALGRIND_CFI_EPILOGUE                                         \
01982          : /*out*/   "=a" (_res)                                       \
01983          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
01984          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
01985       );                                                               \
01986       lval = (__typeof__(lval)) _res;                                  \
01987    } while (0)
01988 
01989 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
01990                                  arg7,arg8)                            \
01991    do {                                                                \
01992       volatile OrigFn        _orig = (orig);                           \
01993       volatile unsigned long _argvec[9];                               \
01994       volatile unsigned long _res;                                     \
01995       _argvec[0] = (unsigned long)_orig.nraddr;                        \
01996       _argvec[1] = (unsigned long)(arg1);                              \
01997       _argvec[2] = (unsigned long)(arg2);                              \
01998       _argvec[3] = (unsigned long)(arg3);                              \
01999       _argvec[4] = (unsigned long)(arg4);                              \
02000       _argvec[5] = (unsigned long)(arg5);                              \
02001       _argvec[6] = (unsigned long)(arg6);                              \
02002       _argvec[7] = (unsigned long)(arg7);                              \
02003       _argvec[8] = (unsigned long)(arg8);                              \
02004       __asm__ volatile(                                                \
02005          VALGRIND_CFI_PROLOGUE                                         \
02006          VALGRIND_ALIGN_STACK                                          \
02007          "subq $128,%%rsp\n\t"                                         \
02008          "pushq 64(%%rax)\n\t"                                         \
02009          "pushq 56(%%rax)\n\t"                                         \
02010          "movq 48(%%rax), %%r9\n\t"                                    \
02011          "movq 40(%%rax), %%r8\n\t"                                    \
02012          "movq 32(%%rax), %%rcx\n\t"                                   \
02013          "movq 24(%%rax), %%rdx\n\t"                                   \
02014          "movq 16(%%rax), %%rsi\n\t"                                   \
02015          "movq 8(%%rax), %%rdi\n\t"                                    \
02016          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
02017          VALGRIND_CALL_NOREDIR_RAX                                     \
02018          VALGRIND_RESTORE_STACK                                        \
02019          VALGRIND_CFI_EPILOGUE                                         \
02020          : /*out*/   "=a" (_res)                                       \
02021          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
02022          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
02023       );                                                               \
02024       lval = (__typeof__(lval)) _res;                                  \
02025    } while (0)
02026 
02027 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
02028                                  arg7,arg8,arg9)                       \
02029    do {                                                                \
02030       volatile OrigFn        _orig = (orig);                           \
02031       volatile unsigned long _argvec[10];                              \
02032       volatile unsigned long _res;                                     \
02033       _argvec[0] = (unsigned long)_orig.nraddr;                        \
02034       _argvec[1] = (unsigned long)(arg1);                              \
02035       _argvec[2] = (unsigned long)(arg2);                              \
02036       _argvec[3] = (unsigned long)(arg3);                              \
02037       _argvec[4] = (unsigned long)(arg4);                              \
02038       _argvec[5] = (unsigned long)(arg5);                              \
02039       _argvec[6] = (unsigned long)(arg6);                              \
02040       _argvec[7] = (unsigned long)(arg7);                              \
02041       _argvec[8] = (unsigned long)(arg8);                              \
02042       _argvec[9] = (unsigned long)(arg9);                              \
02043       __asm__ volatile(                                                \
02044          VALGRIND_CFI_PROLOGUE                                         \
02045          VALGRIND_ALIGN_STACK                                          \
02046          "subq $136,%%rsp\n\t"                                         \
02047          "pushq 72(%%rax)\n\t"                                         \
02048          "pushq 64(%%rax)\n\t"                                         \
02049          "pushq 56(%%rax)\n\t"                                         \
02050          "movq 48(%%rax), %%r9\n\t"                                    \
02051          "movq 40(%%rax), %%r8\n\t"                                    \
02052          "movq 32(%%rax), %%rcx\n\t"                                   \
02053          "movq 24(%%rax), %%rdx\n\t"                                   \
02054          "movq 16(%%rax), %%rsi\n\t"                                   \
02055          "movq 8(%%rax), %%rdi\n\t"                                    \
02056          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
02057          VALGRIND_CALL_NOREDIR_RAX                                     \
02058          VALGRIND_RESTORE_STACK                                        \
02059          VALGRIND_CFI_EPILOGUE                                         \
02060          : /*out*/   "=a" (_res)                                       \
02061          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
02062          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
02063       );                                                               \
02064       lval = (__typeof__(lval)) _res;                                  \
02065    } while (0)
02066 
02067 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
02068                                   arg7,arg8,arg9,arg10)                \
02069    do {                                                                \
02070       volatile OrigFn        _orig = (orig);                           \
02071       volatile unsigned long _argvec[11];                              \
02072       volatile unsigned long _res;                                     \
02073       _argvec[0] = (unsigned long)_orig.nraddr;                        \
02074       _argvec[1] = (unsigned long)(arg1);                              \
02075       _argvec[2] = (unsigned long)(arg2);                              \
02076       _argvec[3] = (unsigned long)(arg3);                              \
02077       _argvec[4] = (unsigned long)(arg4);                              \
02078       _argvec[5] = (unsigned long)(arg5);                              \
02079       _argvec[6] = (unsigned long)(arg6);                              \
02080       _argvec[7] = (unsigned long)(arg7);                              \
02081       _argvec[8] = (unsigned long)(arg8);                              \
02082       _argvec[9] = (unsigned long)(arg9);                              \
02083       _argvec[10] = (unsigned long)(arg10);                            \
02084       __asm__ volatile(                                                \
02085          VALGRIND_CFI_PROLOGUE                                         \
02086          VALGRIND_ALIGN_STACK                                          \
02087          "subq $128,%%rsp\n\t"                                         \
02088          "pushq 80(%%rax)\n\t"                                         \
02089          "pushq 72(%%rax)\n\t"                                         \
02090          "pushq 64(%%rax)\n\t"                                         \
02091          "pushq 56(%%rax)\n\t"                                         \
02092          "movq 48(%%rax), %%r9\n\t"                                    \
02093          "movq 40(%%rax), %%r8\n\t"                                    \
02094          "movq 32(%%rax), %%rcx\n\t"                                   \
02095          "movq 24(%%rax), %%rdx\n\t"                                   \
02096          "movq 16(%%rax), %%rsi\n\t"                                   \
02097          "movq 8(%%rax), %%rdi\n\t"                                    \
02098          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
02099          VALGRIND_CALL_NOREDIR_RAX                                     \
02100          VALGRIND_RESTORE_STACK                                        \
02101          VALGRIND_CFI_EPILOGUE                                         \
02102          : /*out*/   "=a" (_res)                                       \
02103          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
02104          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
02105       );                                                               \
02106       lval = (__typeof__(lval)) _res;                                  \
02107    } while (0)
02108 
02109 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
02110                                   arg7,arg8,arg9,arg10,arg11)          \
02111    do {                                                                \
02112       volatile OrigFn        _orig = (orig);                           \
02113       volatile unsigned long _argvec[12];                              \
02114       volatile unsigned long _res;                                     \
02115       _argvec[0] = (unsigned long)_orig.nraddr;                        \
02116       _argvec[1] = (unsigned long)(arg1);                              \
02117       _argvec[2] = (unsigned long)(arg2);                              \
02118       _argvec[3] = (unsigned long)(arg3);                              \
02119       _argvec[4] = (unsigned long)(arg4);                              \
02120       _argvec[5] = (unsigned long)(arg5);                              \
02121       _argvec[6] = (unsigned long)(arg6);                              \
02122       _argvec[7] = (unsigned long)(arg7);                              \
02123       _argvec[8] = (unsigned long)(arg8);                              \
02124       _argvec[9] = (unsigned long)(arg9);                              \
02125       _argvec[10] = (unsigned long)(arg10);                            \
02126       _argvec[11] = (unsigned long)(arg11);                            \
02127       __asm__ volatile(                                                \
02128          VALGRIND_CFI_PROLOGUE                                         \
02129          VALGRIND_ALIGN_STACK                                          \
02130          "subq $136,%%rsp\n\t"                                         \
02131          "pushq 88(%%rax)\n\t"                                         \
02132          "pushq 80(%%rax)\n\t"                                         \
02133          "pushq 72(%%rax)\n\t"                                         \
02134          "pushq 64(%%rax)\n\t"                                         \
02135          "pushq 56(%%rax)\n\t"                                         \
02136          "movq 48(%%rax), %%r9\n\t"                                    \
02137          "movq 40(%%rax), %%r8\n\t"                                    \
02138          "movq 32(%%rax), %%rcx\n\t"                                   \
02139          "movq 24(%%rax), %%rdx\n\t"                                   \
02140          "movq 16(%%rax), %%rsi\n\t"                                   \
02141          "movq 8(%%rax), %%rdi\n\t"                                    \
02142          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
02143          VALGRIND_CALL_NOREDIR_RAX                                     \
02144          VALGRIND_RESTORE_STACK                                        \
02145          VALGRIND_CFI_EPILOGUE                                         \
02146          : /*out*/   "=a" (_res)                                       \
02147          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
02148          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
02149       );                                                               \
02150       lval = (__typeof__(lval)) _res;                                  \
02151    } while (0)
02152 
02153 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
02154                                 arg7,arg8,arg9,arg10,arg11,arg12)      \
02155    do {                                                                \
02156       volatile OrigFn        _orig = (orig);                           \
02157       volatile unsigned long _argvec[13];                              \
02158       volatile unsigned long _res;                                     \
02159       _argvec[0] = (unsigned long)_orig.nraddr;                        \
02160       _argvec[1] = (unsigned long)(arg1);                              \
02161       _argvec[2] = (unsigned long)(arg2);                              \
02162       _argvec[3] = (unsigned long)(arg3);                              \
02163       _argvec[4] = (unsigned long)(arg4);                              \
02164       _argvec[5] = (unsigned long)(arg5);                              \
02165       _argvec[6] = (unsigned long)(arg6);                              \
02166       _argvec[7] = (unsigned long)(arg7);                              \
02167       _argvec[8] = (unsigned long)(arg8);                              \
02168       _argvec[9] = (unsigned long)(arg9);                              \
02169       _argvec[10] = (unsigned long)(arg10);                            \
02170       _argvec[11] = (unsigned long)(arg11);                            \
02171       _argvec[12] = (unsigned long)(arg12);                            \
02172       __asm__ volatile(                                                \
02173          VALGRIND_CFI_PROLOGUE                                         \
02174          VALGRIND_ALIGN_STACK                                          \
02175          "subq $128,%%rsp\n\t"                                         \
02176          "pushq 96(%%rax)\n\t"                                         \
02177          "pushq 88(%%rax)\n\t"                                         \
02178          "pushq 80(%%rax)\n\t"                                         \
02179          "pushq 72(%%rax)\n\t"                                         \
02180          "pushq 64(%%rax)\n\t"                                         \
02181          "pushq 56(%%rax)\n\t"                                         \
02182          "movq 48(%%rax), %%r9\n\t"                                    \
02183          "movq 40(%%rax), %%r8\n\t"                                    \
02184          "movq 32(%%rax), %%rcx\n\t"                                   \
02185          "movq 24(%%rax), %%rdx\n\t"                                   \
02186          "movq 16(%%rax), %%rsi\n\t"                                   \
02187          "movq 8(%%rax), %%rdi\n\t"                                    \
02188          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
02189          VALGRIND_CALL_NOREDIR_RAX                                     \
02190          VALGRIND_RESTORE_STACK                                        \
02191          VALGRIND_CFI_EPILOGUE                                         \
02192          : /*out*/   "=a" (_res)                                       \
02193          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
02194          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
02195       );                                                               \
02196       lval = (__typeof__(lval)) _res;                                  \
02197    } while (0)
02198 
02199 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */
02200 
02201 /* ------------------------ ppc32-linux ------------------------ */
02202 
02203 #if defined(PLAT_ppc32_linux)
02204 
02205 /* This is useful for finding out about the on-stack stuff:
02206 
02207    extern int f9  ( int,int,int,int,int,int,int,int,int );
02208    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
02209    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
02210    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
02211 
02212    int g9 ( void ) {
02213       return f9(11,22,33,44,55,66,77,88,99);
02214    }
02215    int g10 ( void ) {
02216       return f10(11,22,33,44,55,66,77,88,99,110);
02217    }
02218    int g11 ( void ) {
02219       return f11(11,22,33,44,55,66,77,88,99,110,121);
02220    }
02221    int g12 ( void ) {
02222       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
02223    }
02224 */
02225 
02226 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
02227 
02228 /* These regs are trashed by the hidden call. */
02229 #define __CALLER_SAVED_REGS                                       \
02230    "lr", "ctr", "xer",                                            \
02231    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
02232    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
02233    "r11", "r12", "r13"
02234 
02235 /* Macros to save and align the stack before making a function
02236    call and restore it afterwards as gcc may not keep the stack
02237    pointer aligned if it doesn't realise calls are being made
02238    to other functions. */
02239 
02240 #define VALGRIND_ALIGN_STACK               \
02241       "mr 28,1\n\t"                        \
02242       "rlwinm 1,1,0,0,27\n\t"
02243 #define VALGRIND_RESTORE_STACK             \
02244       "mr 1,28\n\t"
02245 
02246 /* These CALL_FN_ macros assume that on ppc32-linux, 
02247    sizeof(unsigned long) == 4. */
02248 
02249 #define CALL_FN_W_v(lval, orig)                                   \
02250    do {                                                           \
02251       volatile OrigFn        _orig = (orig);                      \
02252       volatile unsigned long _argvec[1];                          \
02253       volatile unsigned long _res;                                \
02254       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02255       __asm__ volatile(                                           \
02256          VALGRIND_ALIGN_STACK                                     \
02257          "mr 11,%1\n\t"                                           \
02258          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02259          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02260          VALGRIND_RESTORE_STACK                                   \
02261          "mr %0,3"                                                \
02262          : /*out*/   "=r" (_res)                                  \
02263          : /*in*/    "r" (&_argvec[0])                            \
02264          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02265       );                                                          \
02266       lval = (__typeof__(lval)) _res;                             \
02267    } while (0)
02268 
02269 #define CALL_FN_W_W(lval, orig, arg1)                             \
02270    do {                                                           \
02271       volatile OrigFn        _orig = (orig);                      \
02272       volatile unsigned long _argvec[2];                          \
02273       volatile unsigned long _res;                                \
02274       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02275       _argvec[1] = (unsigned long)arg1;                           \
02276       __asm__ volatile(                                           \
02277          VALGRIND_ALIGN_STACK                                     \
02278          "mr 11,%1\n\t"                                           \
02279          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02280          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02281          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02282          VALGRIND_RESTORE_STACK                                   \
02283          "mr %0,3"                                                \
02284          : /*out*/   "=r" (_res)                                  \
02285          : /*in*/    "r" (&_argvec[0])                            \
02286          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02287       );                                                          \
02288       lval = (__typeof__(lval)) _res;                             \
02289    } while (0)
02290 
02291 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
02292    do {                                                           \
02293       volatile OrigFn        _orig = (orig);                      \
02294       volatile unsigned long _argvec[3];                          \
02295       volatile unsigned long _res;                                \
02296       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02297       _argvec[1] = (unsigned long)arg1;                           \
02298       _argvec[2] = (unsigned long)arg2;                           \
02299       __asm__ volatile(                                           \
02300          VALGRIND_ALIGN_STACK                                     \
02301          "mr 11,%1\n\t"                                           \
02302          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02303          "lwz 4,8(11)\n\t"                                        \
02304          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02305          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02306          VALGRIND_RESTORE_STACK                                   \
02307          "mr %0,3"                                                \
02308          : /*out*/   "=r" (_res)                                  \
02309          : /*in*/    "r" (&_argvec[0])                            \
02310          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02311       );                                                          \
02312       lval = (__typeof__(lval)) _res;                             \
02313    } while (0)
02314 
02315 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
02316    do {                                                           \
02317       volatile OrigFn        _orig = (orig);                      \
02318       volatile unsigned long _argvec[4];                          \
02319       volatile unsigned long _res;                                \
02320       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02321       _argvec[1] = (unsigned long)arg1;                           \
02322       _argvec[2] = (unsigned long)arg2;                           \
02323       _argvec[3] = (unsigned long)arg3;                           \
02324       __asm__ volatile(                                           \
02325          VALGRIND_ALIGN_STACK                                     \
02326          "mr 11,%1\n\t"                                           \
02327          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02328          "lwz 4,8(11)\n\t"                                        \
02329          "lwz 5,12(11)\n\t"                                       \
02330          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02331          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02332          VALGRIND_RESTORE_STACK                                   \
02333          "mr %0,3"                                                \
02334          : /*out*/   "=r" (_res)                                  \
02335          : /*in*/    "r" (&_argvec[0])                            \
02336          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02337       );                                                          \
02338       lval = (__typeof__(lval)) _res;                             \
02339    } while (0)
02340 
02341 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
02342    do {                                                           \
02343       volatile OrigFn        _orig = (orig);                      \
02344       volatile unsigned long _argvec[5];                          \
02345       volatile unsigned long _res;                                \
02346       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02347       _argvec[1] = (unsigned long)arg1;                           \
02348       _argvec[2] = (unsigned long)arg2;                           \
02349       _argvec[3] = (unsigned long)arg3;                           \
02350       _argvec[4] = (unsigned long)arg4;                           \
02351       __asm__ volatile(                                           \
02352          VALGRIND_ALIGN_STACK                                     \
02353          "mr 11,%1\n\t"                                           \
02354          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02355          "lwz 4,8(11)\n\t"                                        \
02356          "lwz 5,12(11)\n\t"                                       \
02357          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02358          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02359          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02360          VALGRIND_RESTORE_STACK                                   \
02361          "mr %0,3"                                                \
02362          : /*out*/   "=r" (_res)                                  \
02363          : /*in*/    "r" (&_argvec[0])                            \
02364          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02365       );                                                          \
02366       lval = (__typeof__(lval)) _res;                             \
02367    } while (0)
02368 
02369 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
02370    do {                                                           \
02371       volatile OrigFn        _orig = (orig);                      \
02372       volatile unsigned long _argvec[6];                          \
02373       volatile unsigned long _res;                                \
02374       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02375       _argvec[1] = (unsigned long)arg1;                           \
02376       _argvec[2] = (unsigned long)arg2;                           \
02377       _argvec[3] = (unsigned long)arg3;                           \
02378       _argvec[4] = (unsigned long)arg4;                           \
02379       _argvec[5] = (unsigned long)arg5;                           \
02380       __asm__ volatile(                                           \
02381          VALGRIND_ALIGN_STACK                                     \
02382          "mr 11,%1\n\t"                                           \
02383          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02384          "lwz 4,8(11)\n\t"                                        \
02385          "lwz 5,12(11)\n\t"                                       \
02386          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02387          "lwz 7,20(11)\n\t"                                       \
02388          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02389          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02390          VALGRIND_RESTORE_STACK                                   \
02391          "mr %0,3"                                                \
02392          : /*out*/   "=r" (_res)                                  \
02393          : /*in*/    "r" (&_argvec[0])                            \
02394          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02395       );                                                          \
02396       lval = (__typeof__(lval)) _res;                             \
02397    } while (0)
02398 
02399 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
02400    do {                                                           \
02401       volatile OrigFn        _orig = (orig);                      \
02402       volatile unsigned long _argvec[7];                          \
02403       volatile unsigned long _res;                                \
02404       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02405       _argvec[1] = (unsigned long)arg1;                           \
02406       _argvec[2] = (unsigned long)arg2;                           \
02407       _argvec[3] = (unsigned long)arg3;                           \
02408       _argvec[4] = (unsigned long)arg4;                           \
02409       _argvec[5] = (unsigned long)arg5;                           \
02410       _argvec[6] = (unsigned long)arg6;                           \
02411       __asm__ volatile(                                           \
02412          VALGRIND_ALIGN_STACK                                     \
02413          "mr 11,%1\n\t"                                           \
02414          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02415          "lwz 4,8(11)\n\t"                                        \
02416          "lwz 5,12(11)\n\t"                                       \
02417          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02418          "lwz 7,20(11)\n\t"                                       \
02419          "lwz 8,24(11)\n\t"                                       \
02420          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02421          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02422          VALGRIND_RESTORE_STACK                                   \
02423          "mr %0,3"                                                \
02424          : /*out*/   "=r" (_res)                                  \
02425          : /*in*/    "r" (&_argvec[0])                            \
02426          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02427       );                                                          \
02428       lval = (__typeof__(lval)) _res;                             \
02429    } while (0)
02430 
02431 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
02432                                  arg7)                            \
02433    do {                                                           \
02434       volatile OrigFn        _orig = (orig);                      \
02435       volatile unsigned long _argvec[8];                          \
02436       volatile unsigned long _res;                                \
02437       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02438       _argvec[1] = (unsigned long)arg1;                           \
02439       _argvec[2] = (unsigned long)arg2;                           \
02440       _argvec[3] = (unsigned long)arg3;                           \
02441       _argvec[4] = (unsigned long)arg4;                           \
02442       _argvec[5] = (unsigned long)arg5;                           \
02443       _argvec[6] = (unsigned long)arg6;                           \
02444       _argvec[7] = (unsigned long)arg7;                           \
02445       __asm__ volatile(                                           \
02446          VALGRIND_ALIGN_STACK                                     \
02447          "mr 11,%1\n\t"                                           \
02448          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02449          "lwz 4,8(11)\n\t"                                        \
02450          "lwz 5,12(11)\n\t"                                       \
02451          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02452          "lwz 7,20(11)\n\t"                                       \
02453          "lwz 8,24(11)\n\t"                                       \
02454          "lwz 9,28(11)\n\t"                                       \
02455          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02456          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02457          VALGRIND_RESTORE_STACK                                   \
02458          "mr %0,3"                                                \
02459          : /*out*/   "=r" (_res)                                  \
02460          : /*in*/    "r" (&_argvec[0])                            \
02461          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02462       );                                                          \
02463       lval = (__typeof__(lval)) _res;                             \
02464    } while (0)
02465 
02466 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
02467                                  arg7,arg8)                       \
02468    do {                                                           \
02469       volatile OrigFn        _orig = (orig);                      \
02470       volatile unsigned long _argvec[9];                          \
02471       volatile unsigned long _res;                                \
02472       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02473       _argvec[1] = (unsigned long)arg1;                           \
02474       _argvec[2] = (unsigned long)arg2;                           \
02475       _argvec[3] = (unsigned long)arg3;                           \
02476       _argvec[4] = (unsigned long)arg4;                           \
02477       _argvec[5] = (unsigned long)arg5;                           \
02478       _argvec[6] = (unsigned long)arg6;                           \
02479       _argvec[7] = (unsigned long)arg7;                           \
02480       _argvec[8] = (unsigned long)arg8;                           \
02481       __asm__ volatile(                                           \
02482          VALGRIND_ALIGN_STACK                                     \
02483          "mr 11,%1\n\t"                                           \
02484          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02485          "lwz 4,8(11)\n\t"                                        \
02486          "lwz 5,12(11)\n\t"                                       \
02487          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02488          "lwz 7,20(11)\n\t"                                       \
02489          "lwz 8,24(11)\n\t"                                       \
02490          "lwz 9,28(11)\n\t"                                       \
02491          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
02492          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02493          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02494          VALGRIND_RESTORE_STACK                                   \
02495          "mr %0,3"                                                \
02496          : /*out*/   "=r" (_res)                                  \
02497          : /*in*/    "r" (&_argvec[0])                            \
02498          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02499       );                                                          \
02500       lval = (__typeof__(lval)) _res;                             \
02501    } while (0)
02502 
02503 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
02504                                  arg7,arg8,arg9)                  \
02505    do {                                                           \
02506       volatile OrigFn        _orig = (orig);                      \
02507       volatile unsigned long _argvec[10];                         \
02508       volatile unsigned long _res;                                \
02509       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02510       _argvec[1] = (unsigned long)arg1;                           \
02511       _argvec[2] = (unsigned long)arg2;                           \
02512       _argvec[3] = (unsigned long)arg3;                           \
02513       _argvec[4] = (unsigned long)arg4;                           \
02514       _argvec[5] = (unsigned long)arg5;                           \
02515       _argvec[6] = (unsigned long)arg6;                           \
02516       _argvec[7] = (unsigned long)arg7;                           \
02517       _argvec[8] = (unsigned long)arg8;                           \
02518       _argvec[9] = (unsigned long)arg9;                           \
02519       __asm__ volatile(                                           \
02520          VALGRIND_ALIGN_STACK                                     \
02521          "mr 11,%1\n\t"                                           \
02522          "addi 1,1,-16\n\t"                                       \
02523          /* arg9 */                                               \
02524          "lwz 3,36(11)\n\t"                                       \
02525          "stw 3,8(1)\n\t"                                         \
02526          /* args1-8 */                                            \
02527          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02528          "lwz 4,8(11)\n\t"                                        \
02529          "lwz 5,12(11)\n\t"                                       \
02530          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02531          "lwz 7,20(11)\n\t"                                       \
02532          "lwz 8,24(11)\n\t"                                       \
02533          "lwz 9,28(11)\n\t"                                       \
02534          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
02535          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02536          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02537          VALGRIND_RESTORE_STACK                                   \
02538          "mr %0,3"                                                \
02539          : /*out*/   "=r" (_res)                                  \
02540          : /*in*/    "r" (&_argvec[0])                            \
02541          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02542       );                                                          \
02543       lval = (__typeof__(lval)) _res;                             \
02544    } while (0)
02545 
02546 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
02547                                   arg7,arg8,arg9,arg10)           \
02548    do {                                                           \
02549       volatile OrigFn        _orig = (orig);                      \
02550       volatile unsigned long _argvec[11];                         \
02551       volatile unsigned long _res;                                \
02552       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02553       _argvec[1] = (unsigned long)arg1;                           \
02554       _argvec[2] = (unsigned long)arg2;                           \
02555       _argvec[3] = (unsigned long)arg3;                           \
02556       _argvec[4] = (unsigned long)arg4;                           \
02557       _argvec[5] = (unsigned long)arg5;                           \
02558       _argvec[6] = (unsigned long)arg6;                           \
02559       _argvec[7] = (unsigned long)arg7;                           \
02560       _argvec[8] = (unsigned long)arg8;                           \
02561       _argvec[9] = (unsigned long)arg9;                           \
02562       _argvec[10] = (unsigned long)arg10;                         \
02563       __asm__ volatile(                                           \
02564          VALGRIND_ALIGN_STACK                                     \
02565          "mr 11,%1\n\t"                                           \
02566          "addi 1,1,-16\n\t"                                       \
02567          /* arg10 */                                              \
02568          "lwz 3,40(11)\n\t"                                       \
02569          "stw 3,12(1)\n\t"                                        \
02570          /* arg9 */                                               \
02571          "lwz 3,36(11)\n\t"                                       \
02572          "stw 3,8(1)\n\t"                                         \
02573          /* args1-8 */                                            \
02574          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02575          "lwz 4,8(11)\n\t"                                        \
02576          "lwz 5,12(11)\n\t"                                       \
02577          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02578          "lwz 7,20(11)\n\t"                                       \
02579          "lwz 8,24(11)\n\t"                                       \
02580          "lwz 9,28(11)\n\t"                                       \
02581          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
02582          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02583          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02584          VALGRIND_RESTORE_STACK                                   \
02585          "mr %0,3"                                                \
02586          : /*out*/   "=r" (_res)                                  \
02587          : /*in*/    "r" (&_argvec[0])                            \
02588          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02589       );                                                          \
02590       lval = (__typeof__(lval)) _res;                             \
02591    } while (0)
02592 
02593 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
02594                                   arg7,arg8,arg9,arg10,arg11)     \
02595    do {                                                           \
02596       volatile OrigFn        _orig = (orig);                      \
02597       volatile unsigned long _argvec[12];                         \
02598       volatile unsigned long _res;                                \
02599       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02600       _argvec[1] = (unsigned long)arg1;                           \
02601       _argvec[2] = (unsigned long)arg2;                           \
02602       _argvec[3] = (unsigned long)arg3;                           \
02603       _argvec[4] = (unsigned long)arg4;                           \
02604       _argvec[5] = (unsigned long)arg5;                           \
02605       _argvec[6] = (unsigned long)arg6;                           \
02606       _argvec[7] = (unsigned long)arg7;                           \
02607       _argvec[8] = (unsigned long)arg8;                           \
02608       _argvec[9] = (unsigned long)arg9;                           \
02609       _argvec[10] = (unsigned long)arg10;                         \
02610       _argvec[11] = (unsigned long)arg11;                         \
02611       __asm__ volatile(                                           \
02612          VALGRIND_ALIGN_STACK                                     \
02613          "mr 11,%1\n\t"                                           \
02614          "addi 1,1,-32\n\t"                                       \
02615          /* arg11 */                                              \
02616          "lwz 3,44(11)\n\t"                                       \
02617          "stw 3,16(1)\n\t"                                        \
02618          /* arg10 */                                              \
02619          "lwz 3,40(11)\n\t"                                       \
02620          "stw 3,12(1)\n\t"                                        \
02621          /* arg9 */                                               \
02622          "lwz 3,36(11)\n\t"                                       \
02623          "stw 3,8(1)\n\t"                                         \
02624          /* args1-8 */                                            \
02625          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02626          "lwz 4,8(11)\n\t"                                        \
02627          "lwz 5,12(11)\n\t"                                       \
02628          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02629          "lwz 7,20(11)\n\t"                                       \
02630          "lwz 8,24(11)\n\t"                                       \
02631          "lwz 9,28(11)\n\t"                                       \
02632          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
02633          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02634          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02635          VALGRIND_RESTORE_STACK                                   \
02636          "mr %0,3"                                                \
02637          : /*out*/   "=r" (_res)                                  \
02638          : /*in*/    "r" (&_argvec[0])                            \
02639          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02640       );                                                          \
02641       lval = (__typeof__(lval)) _res;                             \
02642    } while (0)
02643 
02644 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
02645                                 arg7,arg8,arg9,arg10,arg11,arg12) \
02646    do {                                                           \
02647       volatile OrigFn        _orig = (orig);                      \
02648       volatile unsigned long _argvec[13];                         \
02649       volatile unsigned long _res;                                \
02650       _argvec[0] = (unsigned long)_orig.nraddr;                   \
02651       _argvec[1] = (unsigned long)arg1;                           \
02652       _argvec[2] = (unsigned long)arg2;                           \
02653       _argvec[3] = (unsigned long)arg3;                           \
02654       _argvec[4] = (unsigned long)arg4;                           \
02655       _argvec[5] = (unsigned long)arg5;                           \
02656       _argvec[6] = (unsigned long)arg6;                           \
02657       _argvec[7] = (unsigned long)arg7;                           \
02658       _argvec[8] = (unsigned long)arg8;                           \
02659       _argvec[9] = (unsigned long)arg9;                           \
02660       _argvec[10] = (unsigned long)arg10;                         \
02661       _argvec[11] = (unsigned long)arg11;                         \
02662       _argvec[12] = (unsigned long)arg12;                         \
02663       __asm__ volatile(                                           \
02664          VALGRIND_ALIGN_STACK                                     \
02665          "mr 11,%1\n\t"                                           \
02666          "addi 1,1,-32\n\t"                                       \
02667          /* arg12 */                                              \
02668          "lwz 3,48(11)\n\t"                                       \
02669          "stw 3,20(1)\n\t"                                        \
02670          /* arg11 */                                              \
02671          "lwz 3,44(11)\n\t"                                       \
02672          "stw 3,16(1)\n\t"                                        \
02673          /* arg10 */                                              \
02674          "lwz 3,40(11)\n\t"                                       \
02675          "stw 3,12(1)\n\t"                                        \
02676          /* arg9 */                                               \
02677          "lwz 3,36(11)\n\t"                                       \
02678          "stw 3,8(1)\n\t"                                         \
02679          /* args1-8 */                                            \
02680          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
02681          "lwz 4,8(11)\n\t"                                        \
02682          "lwz 5,12(11)\n\t"                                       \
02683          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
02684          "lwz 7,20(11)\n\t"                                       \
02685          "lwz 8,24(11)\n\t"                                       \
02686          "lwz 9,28(11)\n\t"                                       \
02687          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
02688          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
02689          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02690          VALGRIND_RESTORE_STACK                                   \
02691          "mr %0,3"                                                \
02692          : /*out*/   "=r" (_res)                                  \
02693          : /*in*/    "r" (&_argvec[0])                            \
02694          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02695       );                                                          \
02696       lval = (__typeof__(lval)) _res;                             \
02697    } while (0)
02698 
02699 #endif /* PLAT_ppc32_linux */
02700 
02701 /* ------------------------ ppc64-linux ------------------------ */
02702 
02703 #if defined(PLAT_ppc64be_linux)
02704 
02705 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
02706 
02707 /* These regs are trashed by the hidden call. */
02708 #define __CALLER_SAVED_REGS                                       \
02709    "lr", "ctr", "xer",                                            \
02710    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
02711    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
02712    "r11", "r12", "r13"
02713 
02714 /* Macros to save and align the stack before making a function
02715    call and restore it afterwards as gcc may not keep the stack
02716    pointer aligned if it doesn't realise calls are being made
02717    to other functions. */
02718 
02719 #define VALGRIND_ALIGN_STACK               \
02720       "mr 28,1\n\t"                        \
02721       "rldicr 1,1,0,59\n\t"
02722 #define VALGRIND_RESTORE_STACK             \
02723       "mr 1,28\n\t"
02724 
02725 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
02726    long) == 8. */
02727 
02728 #define CALL_FN_W_v(lval, orig)                                   \
02729    do {                                                           \
02730       volatile OrigFn        _orig = (orig);                      \
02731       volatile unsigned long _argvec[3+0];                        \
02732       volatile unsigned long _res;                                \
02733       /* _argvec[0] holds current r2 across the call */           \
02734       _argvec[1] = (unsigned long)_orig.r2;                       \
02735       _argvec[2] = (unsigned long)_orig.nraddr;                   \
02736       __asm__ volatile(                                           \
02737          VALGRIND_ALIGN_STACK                                     \
02738          "mr 11,%1\n\t"                                           \
02739          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02740          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02741          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02742          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02743          "mr 11,%1\n\t"                                           \
02744          "mr %0,3\n\t"                                            \
02745          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02746          VALGRIND_RESTORE_STACK                                   \
02747          : /*out*/   "=r" (_res)                                  \
02748          : /*in*/    "r" (&_argvec[2])                            \
02749          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02750       );                                                          \
02751       lval = (__typeof__(lval)) _res;                             \
02752    } while (0)
02753 
02754 #define CALL_FN_W_W(lval, orig, arg1)                             \
02755    do {                                                           \
02756       volatile OrigFn        _orig = (orig);                      \
02757       volatile unsigned long _argvec[3+1];                        \
02758       volatile unsigned long _res;                                \
02759       /* _argvec[0] holds current r2 across the call */           \
02760       _argvec[1]   = (unsigned long)_orig.r2;                     \
02761       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02762       _argvec[2+1] = (unsigned long)arg1;                         \
02763       __asm__ volatile(                                           \
02764          VALGRIND_ALIGN_STACK                                     \
02765          "mr 11,%1\n\t"                                           \
02766          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02767          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02768          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02769          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02770          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02771          "mr 11,%1\n\t"                                           \
02772          "mr %0,3\n\t"                                            \
02773          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02774          VALGRIND_RESTORE_STACK                                   \
02775          : /*out*/   "=r" (_res)                                  \
02776          : /*in*/    "r" (&_argvec[2])                            \
02777          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02778       );                                                          \
02779       lval = (__typeof__(lval)) _res;                             \
02780    } while (0)
02781 
02782 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
02783    do {                                                           \
02784       volatile OrigFn        _orig = (orig);                      \
02785       volatile unsigned long _argvec[3+2];                        \
02786       volatile unsigned long _res;                                \
02787       /* _argvec[0] holds current r2 across the call */           \
02788       _argvec[1]   = (unsigned long)_orig.r2;                     \
02789       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02790       _argvec[2+1] = (unsigned long)arg1;                         \
02791       _argvec[2+2] = (unsigned long)arg2;                         \
02792       __asm__ volatile(                                           \
02793          VALGRIND_ALIGN_STACK                                     \
02794          "mr 11,%1\n\t"                                           \
02795          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02796          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02797          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02798          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
02799          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02800          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02801          "mr 11,%1\n\t"                                           \
02802          "mr %0,3\n\t"                                            \
02803          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02804          VALGRIND_RESTORE_STACK                                   \
02805          : /*out*/   "=r" (_res)                                  \
02806          : /*in*/    "r" (&_argvec[2])                            \
02807          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02808       );                                                          \
02809       lval = (__typeof__(lval)) _res;                             \
02810    } while (0)
02811 
02812 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
02813    do {                                                           \
02814       volatile OrigFn        _orig = (orig);                      \
02815       volatile unsigned long _argvec[3+3];                        \
02816       volatile unsigned long _res;                                \
02817       /* _argvec[0] holds current r2 across the call */           \
02818       _argvec[1]   = (unsigned long)_orig.r2;                     \
02819       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02820       _argvec[2+1] = (unsigned long)arg1;                         \
02821       _argvec[2+2] = (unsigned long)arg2;                         \
02822       _argvec[2+3] = (unsigned long)arg3;                         \
02823       __asm__ volatile(                                           \
02824          VALGRIND_ALIGN_STACK                                     \
02825          "mr 11,%1\n\t"                                           \
02826          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02827          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02828          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02829          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
02830          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
02831          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02832          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02833          "mr 11,%1\n\t"                                           \
02834          "mr %0,3\n\t"                                            \
02835          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02836          VALGRIND_RESTORE_STACK                                   \
02837          : /*out*/   "=r" (_res)                                  \
02838          : /*in*/    "r" (&_argvec[2])                            \
02839          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02840       );                                                          \
02841       lval = (__typeof__(lval)) _res;                             \
02842    } while (0)
02843 
02844 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
02845    do {                                                           \
02846       volatile OrigFn        _orig = (orig);                      \
02847       volatile unsigned long _argvec[3+4];                        \
02848       volatile unsigned long _res;                                \
02849       /* _argvec[0] holds current r2 across the call */           \
02850       _argvec[1]   = (unsigned long)_orig.r2;                     \
02851       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02852       _argvec[2+1] = (unsigned long)arg1;                         \
02853       _argvec[2+2] = (unsigned long)arg2;                         \
02854       _argvec[2+3] = (unsigned long)arg3;                         \
02855       _argvec[2+4] = (unsigned long)arg4;                         \
02856       __asm__ volatile(                                           \
02857          VALGRIND_ALIGN_STACK                                     \
02858          "mr 11,%1\n\t"                                           \
02859          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02860          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02861          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02862          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
02863          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
02864          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
02865          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02866          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02867          "mr 11,%1\n\t"                                           \
02868          "mr %0,3\n\t"                                            \
02869          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02870          VALGRIND_RESTORE_STACK                                   \
02871          : /*out*/   "=r" (_res)                                  \
02872          : /*in*/    "r" (&_argvec[2])                            \
02873          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02874       );                                                          \
02875       lval = (__typeof__(lval)) _res;                             \
02876    } while (0)
02877 
02878 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
02879    do {                                                           \
02880       volatile OrigFn        _orig = (orig);                      \
02881       volatile unsigned long _argvec[3+5];                        \
02882       volatile unsigned long _res;                                \
02883       /* _argvec[0] holds current r2 across the call */           \
02884       _argvec[1]   = (unsigned long)_orig.r2;                     \
02885       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02886       _argvec[2+1] = (unsigned long)arg1;                         \
02887       _argvec[2+2] = (unsigned long)arg2;                         \
02888       _argvec[2+3] = (unsigned long)arg3;                         \
02889       _argvec[2+4] = (unsigned long)arg4;                         \
02890       _argvec[2+5] = (unsigned long)arg5;                         \
02891       __asm__ volatile(                                           \
02892          VALGRIND_ALIGN_STACK                                     \
02893          "mr 11,%1\n\t"                                           \
02894          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02895          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02896          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02897          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
02898          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
02899          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
02900          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
02901          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02902          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02903          "mr 11,%1\n\t"                                           \
02904          "mr %0,3\n\t"                                            \
02905          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02906          VALGRIND_RESTORE_STACK                                   \
02907          : /*out*/   "=r" (_res)                                  \
02908          : /*in*/    "r" (&_argvec[2])                            \
02909          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02910       );                                                          \
02911       lval = (__typeof__(lval)) _res;                             \
02912    } while (0)
02913 
02914 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
02915    do {                                                           \
02916       volatile OrigFn        _orig = (orig);                      \
02917       volatile unsigned long _argvec[3+6];                        \
02918       volatile unsigned long _res;                                \
02919       /* _argvec[0] holds current r2 across the call */           \
02920       _argvec[1]   = (unsigned long)_orig.r2;                     \
02921       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02922       _argvec[2+1] = (unsigned long)arg1;                         \
02923       _argvec[2+2] = (unsigned long)arg2;                         \
02924       _argvec[2+3] = (unsigned long)arg3;                         \
02925       _argvec[2+4] = (unsigned long)arg4;                         \
02926       _argvec[2+5] = (unsigned long)arg5;                         \
02927       _argvec[2+6] = (unsigned long)arg6;                         \
02928       __asm__ volatile(                                           \
02929          VALGRIND_ALIGN_STACK                                     \
02930          "mr 11,%1\n\t"                                           \
02931          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02932          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02933          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02934          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
02935          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
02936          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
02937          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
02938          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
02939          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02940          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02941          "mr 11,%1\n\t"                                           \
02942          "mr %0,3\n\t"                                            \
02943          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02944          VALGRIND_RESTORE_STACK                                   \
02945          : /*out*/   "=r" (_res)                                  \
02946          : /*in*/    "r" (&_argvec[2])                            \
02947          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02948       );                                                          \
02949       lval = (__typeof__(lval)) _res;                             \
02950    } while (0)
02951 
02952 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
02953                                  arg7)                            \
02954    do {                                                           \
02955       volatile OrigFn        _orig = (orig);                      \
02956       volatile unsigned long _argvec[3+7];                        \
02957       volatile unsigned long _res;                                \
02958       /* _argvec[0] holds current r2 across the call */           \
02959       _argvec[1]   = (unsigned long)_orig.r2;                     \
02960       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
02961       _argvec[2+1] = (unsigned long)arg1;                         \
02962       _argvec[2+2] = (unsigned long)arg2;                         \
02963       _argvec[2+3] = (unsigned long)arg3;                         \
02964       _argvec[2+4] = (unsigned long)arg4;                         \
02965       _argvec[2+5] = (unsigned long)arg5;                         \
02966       _argvec[2+6] = (unsigned long)arg6;                         \
02967       _argvec[2+7] = (unsigned long)arg7;                         \
02968       __asm__ volatile(                                           \
02969          VALGRIND_ALIGN_STACK                                     \
02970          "mr 11,%1\n\t"                                           \
02971          "std 2,-16(11)\n\t"  /* save tocptr */                   \
02972          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
02973          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
02974          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
02975          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
02976          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
02977          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
02978          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
02979          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
02980          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
02981          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
02982          "mr 11,%1\n\t"                                           \
02983          "mr %0,3\n\t"                                            \
02984          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
02985          VALGRIND_RESTORE_STACK                                   \
02986          : /*out*/   "=r" (_res)                                  \
02987          : /*in*/    "r" (&_argvec[2])                            \
02988          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
02989       );                                                          \
02990       lval = (__typeof__(lval)) _res;                             \
02991    } while (0)
02992 
02993 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
02994                                  arg7,arg8)                       \
02995    do {                                                           \
02996       volatile OrigFn        _orig = (orig);                      \
02997       volatile unsigned long _argvec[3+8];                        \
02998       volatile unsigned long _res;                                \
02999       /* _argvec[0] holds current r2 across the call */           \
03000       _argvec[1]   = (unsigned long)_orig.r2;                     \
03001       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03002       _argvec[2+1] = (unsigned long)arg1;                         \
03003       _argvec[2+2] = (unsigned long)arg2;                         \
03004       _argvec[2+3] = (unsigned long)arg3;                         \
03005       _argvec[2+4] = (unsigned long)arg4;                         \
03006       _argvec[2+5] = (unsigned long)arg5;                         \
03007       _argvec[2+6] = (unsigned long)arg6;                         \
03008       _argvec[2+7] = (unsigned long)arg7;                         \
03009       _argvec[2+8] = (unsigned long)arg8;                         \
03010       __asm__ volatile(                                           \
03011          VALGRIND_ALIGN_STACK                                     \
03012          "mr 11,%1\n\t"                                           \
03013          "std 2,-16(11)\n\t"  /* save tocptr */                   \
03014          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
03015          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
03016          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
03017          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
03018          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
03019          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
03020          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
03021          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
03022          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
03023          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
03024          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
03025          "mr 11,%1\n\t"                                           \
03026          "mr %0,3\n\t"                                            \
03027          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
03028          VALGRIND_RESTORE_STACK                                   \
03029          : /*out*/   "=r" (_res)                                  \
03030          : /*in*/    "r" (&_argvec[2])                            \
03031          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03032       );                                                          \
03033       lval = (__typeof__(lval)) _res;                             \
03034    } while (0)
03035 
03036 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
03037                                  arg7,arg8,arg9)                  \
03038    do {                                                           \
03039       volatile OrigFn        _orig = (orig);                      \
03040       volatile unsigned long _argvec[3+9];                        \
03041       volatile unsigned long _res;                                \
03042       /* _argvec[0] holds current r2 across the call */           \
03043       _argvec[1]   = (unsigned long)_orig.r2;                     \
03044       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03045       _argvec[2+1] = (unsigned long)arg1;                         \
03046       _argvec[2+2] = (unsigned long)arg2;                         \
03047       _argvec[2+3] = (unsigned long)arg3;                         \
03048       _argvec[2+4] = (unsigned long)arg4;                         \
03049       _argvec[2+5] = (unsigned long)arg5;                         \
03050       _argvec[2+6] = (unsigned long)arg6;                         \
03051       _argvec[2+7] = (unsigned long)arg7;                         \
03052       _argvec[2+8] = (unsigned long)arg8;                         \
03053       _argvec[2+9] = (unsigned long)arg9;                         \
03054       __asm__ volatile(                                           \
03055          VALGRIND_ALIGN_STACK                                     \
03056          "mr 11,%1\n\t"                                           \
03057          "std 2,-16(11)\n\t"  /* save tocptr */                   \
03058          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
03059          "addi 1,1,-128\n\t"  /* expand stack frame */            \
03060          /* arg9 */                                               \
03061          "ld  3,72(11)\n\t"                                       \
03062          "std 3,112(1)\n\t"                                       \
03063          /* args1-8 */                                            \
03064          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
03065          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
03066          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
03067          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
03068          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
03069          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
03070          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
03071          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
03072          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
03073          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
03074          "mr 11,%1\n\t"                                           \
03075          "mr %0,3\n\t"                                            \
03076          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
03077          VALGRIND_RESTORE_STACK                                   \
03078          : /*out*/   "=r" (_res)                                  \
03079          : /*in*/    "r" (&_argvec[2])                            \
03080          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03081       );                                                          \
03082       lval = (__typeof__(lval)) _res;                             \
03083    } while (0)
03084 
03085 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
03086                                   arg7,arg8,arg9,arg10)           \
03087    do {                                                           \
03088       volatile OrigFn        _orig = (orig);                      \
03089       volatile unsigned long _argvec[3+10];                       \
03090       volatile unsigned long _res;                                \
03091       /* _argvec[0] holds current r2 across the call */           \
03092       _argvec[1]   = (unsigned long)_orig.r2;                     \
03093       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03094       _argvec[2+1] = (unsigned long)arg1;                         \
03095       _argvec[2+2] = (unsigned long)arg2;                         \
03096       _argvec[2+3] = (unsigned long)arg3;                         \
03097       _argvec[2+4] = (unsigned long)arg4;                         \
03098       _argvec[2+5] = (unsigned long)arg5;                         \
03099       _argvec[2+6] = (unsigned long)arg6;                         \
03100       _argvec[2+7] = (unsigned long)arg7;                         \
03101       _argvec[2+8] = (unsigned long)arg8;                         \
03102       _argvec[2+9] = (unsigned long)arg9;                         \
03103       _argvec[2+10] = (unsigned long)arg10;                       \
03104       __asm__ volatile(                                           \
03105          VALGRIND_ALIGN_STACK                                     \
03106          "mr 11,%1\n\t"                                           \
03107          "std 2,-16(11)\n\t"  /* save tocptr */                   \
03108          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
03109          "addi 1,1,-128\n\t"  /* expand stack frame */            \
03110          /* arg10 */                                              \
03111          "ld  3,80(11)\n\t"                                       \
03112          "std 3,120(1)\n\t"                                       \
03113          /* arg9 */                                               \
03114          "ld  3,72(11)\n\t"                                       \
03115          "std 3,112(1)\n\t"                                       \
03116          /* args1-8 */                                            \
03117          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
03118          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
03119          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
03120          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
03121          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
03122          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
03123          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
03124          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
03125          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
03126          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
03127          "mr 11,%1\n\t"                                           \
03128          "mr %0,3\n\t"                                            \
03129          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
03130          VALGRIND_RESTORE_STACK                                   \
03131          : /*out*/   "=r" (_res)                                  \
03132          : /*in*/    "r" (&_argvec[2])                            \
03133          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03134       );                                                          \
03135       lval = (__typeof__(lval)) _res;                             \
03136    } while (0)
03137 
03138 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
03139                                   arg7,arg8,arg9,arg10,arg11)     \
03140    do {                                                           \
03141       volatile OrigFn        _orig = (orig);                      \
03142       volatile unsigned long _argvec[3+11];                       \
03143       volatile unsigned long _res;                                \
03144       /* _argvec[0] holds current r2 across the call */           \
03145       _argvec[1]   = (unsigned long)_orig.r2;                     \
03146       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03147       _argvec[2+1] = (unsigned long)arg1;                         \
03148       _argvec[2+2] = (unsigned long)arg2;                         \
03149       _argvec[2+3] = (unsigned long)arg3;                         \
03150       _argvec[2+4] = (unsigned long)arg4;                         \
03151       _argvec[2+5] = (unsigned long)arg5;                         \
03152       _argvec[2+6] = (unsigned long)arg6;                         \
03153       _argvec[2+7] = (unsigned long)arg7;                         \
03154       _argvec[2+8] = (unsigned long)arg8;                         \
03155       _argvec[2+9] = (unsigned long)arg9;                         \
03156       _argvec[2+10] = (unsigned long)arg10;                       \
03157       _argvec[2+11] = (unsigned long)arg11;                       \
03158       __asm__ volatile(                                           \
03159          VALGRIND_ALIGN_STACK                                     \
03160          "mr 11,%1\n\t"                                           \
03161          "std 2,-16(11)\n\t"  /* save tocptr */                   \
03162          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
03163          "addi 1,1,-144\n\t"  /* expand stack frame */            \
03164          /* arg11 */                                              \
03165          "ld  3,88(11)\n\t"                                       \
03166          "std 3,128(1)\n\t"                                       \
03167          /* arg10 */                                              \
03168          "ld  3,80(11)\n\t"                                       \
03169          "std 3,120(1)\n\t"                                       \
03170          /* arg9 */                                               \
03171          "ld  3,72(11)\n\t"                                       \
03172          "std 3,112(1)\n\t"                                       \
03173          /* args1-8 */                                            \
03174          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
03175          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
03176          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
03177          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
03178          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
03179          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
03180          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
03181          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
03182          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
03183          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
03184          "mr 11,%1\n\t"                                           \
03185          "mr %0,3\n\t"                                            \
03186          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
03187          VALGRIND_RESTORE_STACK                                   \
03188          : /*out*/   "=r" (_res)                                  \
03189          : /*in*/    "r" (&_argvec[2])                            \
03190          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03191       );                                                          \
03192       lval = (__typeof__(lval)) _res;                             \
03193    } while (0)
03194 
03195 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
03196                                 arg7,arg8,arg9,arg10,arg11,arg12) \
03197    do {                                                           \
03198       volatile OrigFn        _orig = (orig);                      \
03199       volatile unsigned long _argvec[3+12];                       \
03200       volatile unsigned long _res;                                \
03201       /* _argvec[0] holds current r2 across the call */           \
03202       _argvec[1]   = (unsigned long)_orig.r2;                     \
03203       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03204       _argvec[2+1] = (unsigned long)arg1;                         \
03205       _argvec[2+2] = (unsigned long)arg2;                         \
03206       _argvec[2+3] = (unsigned long)arg3;                         \
03207       _argvec[2+4] = (unsigned long)arg4;                         \
03208       _argvec[2+5] = (unsigned long)arg5;                         \
03209       _argvec[2+6] = (unsigned long)arg6;                         \
03210       _argvec[2+7] = (unsigned long)arg7;                         \
03211       _argvec[2+8] = (unsigned long)arg8;                         \
03212       _argvec[2+9] = (unsigned long)arg9;                         \
03213       _argvec[2+10] = (unsigned long)arg10;                       \
03214       _argvec[2+11] = (unsigned long)arg11;                       \
03215       _argvec[2+12] = (unsigned long)arg12;                       \
03216       __asm__ volatile(                                           \
03217          VALGRIND_ALIGN_STACK                                     \
03218          "mr 11,%1\n\t"                                           \
03219          "std 2,-16(11)\n\t"  /* save tocptr */                   \
03220          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
03221          "addi 1,1,-144\n\t"  /* expand stack frame */            \
03222          /* arg12 */                                              \
03223          "ld  3,96(11)\n\t"                                       \
03224          "std 3,136(1)\n\t"                                       \
03225          /* arg11 */                                              \
03226          "ld  3,88(11)\n\t"                                       \
03227          "std 3,128(1)\n\t"                                       \
03228          /* arg10 */                                              \
03229          "ld  3,80(11)\n\t"                                       \
03230          "std 3,120(1)\n\t"                                       \
03231          /* arg9 */                                               \
03232          "ld  3,72(11)\n\t"                                       \
03233          "std 3,112(1)\n\t"                                       \
03234          /* args1-8 */                                            \
03235          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
03236          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
03237          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
03238          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
03239          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
03240          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
03241          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
03242          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
03243          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
03244          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
03245          "mr 11,%1\n\t"                                           \
03246          "mr %0,3\n\t"                                            \
03247          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
03248          VALGRIND_RESTORE_STACK                                   \
03249          : /*out*/   "=r" (_res)                                  \
03250          : /*in*/    "r" (&_argvec[2])                            \
03251          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03252       );                                                          \
03253       lval = (__typeof__(lval)) _res;                             \
03254    } while (0)
03255 
03256 #endif /* PLAT_ppc64be_linux */
03257 
03258 /* ------------------------- ppc64le-linux ----------------------- */
03259 #if defined(PLAT_ppc64le_linux)
03260 
03261 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
03262 
03263 /* These regs are trashed by the hidden call. */
03264 #define __CALLER_SAVED_REGS                                       \
03265    "lr", "ctr", "xer",                                            \
03266    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
03267    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
03268    "r11", "r12", "r13"
03269 
03270 /* Macros to save and align the stack before making a function
03271    call and restore it afterwards as gcc may not keep the stack
03272    pointer aligned if it doesn't realise calls are being made
03273    to other functions. */
03274 
03275 #define VALGRIND_ALIGN_STACK               \
03276       "mr 28,1\n\t"                        \
03277       "rldicr 1,1,0,59\n\t"
03278 #define VALGRIND_RESTORE_STACK             \
03279       "mr 1,28\n\t"
03280 
03281 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
03282    long) == 8. */
03283 
03284 #define CALL_FN_W_v(lval, orig)                                   \
03285    do {                                                           \
03286       volatile OrigFn        _orig = (orig);                      \
03287       volatile unsigned long _argvec[3+0];                        \
03288       volatile unsigned long _res;                                \
03289       /* _argvec[0] holds current r2 across the call */           \
03290       _argvec[1] = (unsigned long)_orig.r2;                       \
03291       _argvec[2] = (unsigned long)_orig.nraddr;                   \
03292       __asm__ volatile(                                           \
03293          VALGRIND_ALIGN_STACK                                     \
03294          "mr 12,%1\n\t"                                           \
03295          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03296          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03297          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03298          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03299          "mr 12,%1\n\t"                                           \
03300          "mr %0,3\n\t"                                            \
03301          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03302          VALGRIND_RESTORE_STACK                                   \
03303          : /*out*/   "=r" (_res)                                  \
03304          : /*in*/    "r" (&_argvec[2])                            \
03305          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03306       );                                                          \
03307       lval = (__typeof__(lval)) _res;                             \
03308    } while (0)
03309 
03310 #define CALL_FN_W_W(lval, orig, arg1)                             \
03311    do {                                                           \
03312       volatile OrigFn        _orig = (orig);                      \
03313       volatile unsigned long _argvec[3+1];                        \
03314       volatile unsigned long _res;                                \
03315       /* _argvec[0] holds current r2 across the call */           \
03316       _argvec[1]   = (unsigned long)_orig.r2;                     \
03317       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03318       _argvec[2+1] = (unsigned long)arg1;                         \
03319       __asm__ volatile(                                           \
03320          VALGRIND_ALIGN_STACK                                     \
03321          "mr 12,%1\n\t"                                           \
03322          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03323          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03324          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03325          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03326          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03327          "mr 12,%1\n\t"                                           \
03328          "mr %0,3\n\t"                                            \
03329          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03330          VALGRIND_RESTORE_STACK                                   \
03331          : /*out*/   "=r" (_res)                                  \
03332          : /*in*/    "r" (&_argvec[2])                            \
03333          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03334       );                                                          \
03335       lval = (__typeof__(lval)) _res;                             \
03336    } while (0)
03337 
03338 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
03339    do {                                                           \
03340       volatile OrigFn        _orig = (orig);                      \
03341       volatile unsigned long _argvec[3+2];                        \
03342       volatile unsigned long _res;                                \
03343       /* _argvec[0] holds current r2 across the call */           \
03344       _argvec[1]   = (unsigned long)_orig.r2;                     \
03345       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03346       _argvec[2+1] = (unsigned long)arg1;                         \
03347       _argvec[2+2] = (unsigned long)arg2;                         \
03348       __asm__ volatile(                                           \
03349          VALGRIND_ALIGN_STACK                                     \
03350          "mr 12,%1\n\t"                                           \
03351          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03352          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03353          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03354          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03355          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03356          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03357          "mr 12,%1\n\t"                                           \
03358          "mr %0,3\n\t"                                            \
03359          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03360          VALGRIND_RESTORE_STACK                                   \
03361          : /*out*/   "=r" (_res)                                  \
03362          : /*in*/    "r" (&_argvec[2])                            \
03363          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03364       );                                                          \
03365       lval = (__typeof__(lval)) _res;                             \
03366    } while (0)
03367 
03368 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
03369    do {                                                           \
03370       volatile OrigFn        _orig = (orig);                      \
03371       volatile unsigned long _argvec[3+3];                        \
03372       volatile unsigned long _res;                                \
03373       /* _argvec[0] holds current r2 across the call */           \
03374       _argvec[1]   = (unsigned long)_orig.r2;                     \
03375       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03376       _argvec[2+1] = (unsigned long)arg1;                         \
03377       _argvec[2+2] = (unsigned long)arg2;                         \
03378       _argvec[2+3] = (unsigned long)arg3;                         \
03379       __asm__ volatile(                                           \
03380          VALGRIND_ALIGN_STACK                                     \
03381          "mr 12,%1\n\t"                                           \
03382          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03383          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03384          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03385          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03386          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03387          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03388          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03389          "mr 12,%1\n\t"                                           \
03390          "mr %0,3\n\t"                                            \
03391          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03392          VALGRIND_RESTORE_STACK                                   \
03393          : /*out*/   "=r" (_res)                                  \
03394          : /*in*/    "r" (&_argvec[2])                            \
03395          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03396       );                                                          \
03397       lval = (__typeof__(lval)) _res;                             \
03398    } while (0)
03399 
03400 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
03401    do {                                                           \
03402       volatile OrigFn        _orig = (orig);                      \
03403       volatile unsigned long _argvec[3+4];                        \
03404       volatile unsigned long _res;                                \
03405       /* _argvec[0] holds current r2 across the call */           \
03406       _argvec[1]   = (unsigned long)_orig.r2;                     \
03407       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03408       _argvec[2+1] = (unsigned long)arg1;                         \
03409       _argvec[2+2] = (unsigned long)arg2;                         \
03410       _argvec[2+3] = (unsigned long)arg3;                         \
03411       _argvec[2+4] = (unsigned long)arg4;                         \
03412       __asm__ volatile(                                           \
03413          VALGRIND_ALIGN_STACK                                     \
03414          "mr 12,%1\n\t"                                           \
03415          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03416          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03417          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03418          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03419          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03420          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03421          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03422          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03423          "mr 12,%1\n\t"                                           \
03424          "mr %0,3\n\t"                                            \
03425          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03426          VALGRIND_RESTORE_STACK                                   \
03427          : /*out*/   "=r" (_res)                                  \
03428          : /*in*/    "r" (&_argvec[2])                            \
03429          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03430       );                                                          \
03431       lval = (__typeof__(lval)) _res;                             \
03432    } while (0)
03433 
03434 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
03435    do {                                                           \
03436       volatile OrigFn        _orig = (orig);                      \
03437       volatile unsigned long _argvec[3+5];                        \
03438       volatile unsigned long _res;                                \
03439       /* _argvec[0] holds current r2 across the call */           \
03440       _argvec[1]   = (unsigned long)_orig.r2;                     \
03441       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03442       _argvec[2+1] = (unsigned long)arg1;                         \
03443       _argvec[2+2] = (unsigned long)arg2;                         \
03444       _argvec[2+3] = (unsigned long)arg3;                         \
03445       _argvec[2+4] = (unsigned long)arg4;                         \
03446       _argvec[2+5] = (unsigned long)arg5;                         \
03447       __asm__ volatile(                                           \
03448          VALGRIND_ALIGN_STACK                                     \
03449          "mr 12,%1\n\t"                                           \
03450          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03451          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03452          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03453          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03454          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03455          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03456          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03457          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03458          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03459          "mr 12,%1\n\t"                                           \
03460          "mr %0,3\n\t"                                            \
03461          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03462          VALGRIND_RESTORE_STACK                                   \
03463          : /*out*/   "=r" (_res)                                  \
03464          : /*in*/    "r" (&_argvec[2])                            \
03465          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03466       );                                                          \
03467       lval = (__typeof__(lval)) _res;                             \
03468    } while (0)
03469 
03470 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
03471    do {                                                           \
03472       volatile OrigFn        _orig = (orig);                      \
03473       volatile unsigned long _argvec[3+6];                        \
03474       volatile unsigned long _res;                                \
03475       /* _argvec[0] holds current r2 across the call */           \
03476       _argvec[1]   = (unsigned long)_orig.r2;                     \
03477       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03478       _argvec[2+1] = (unsigned long)arg1;                         \
03479       _argvec[2+2] = (unsigned long)arg2;                         \
03480       _argvec[2+3] = (unsigned long)arg3;                         \
03481       _argvec[2+4] = (unsigned long)arg4;                         \
03482       _argvec[2+5] = (unsigned long)arg5;                         \
03483       _argvec[2+6] = (unsigned long)arg6;                         \
03484       __asm__ volatile(                                           \
03485          VALGRIND_ALIGN_STACK                                     \
03486          "mr 12,%1\n\t"                                           \
03487          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03488          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03489          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03490          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03491          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03492          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03493          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03494          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03495          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03496          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03497          "mr 12,%1\n\t"                                           \
03498          "mr %0,3\n\t"                                            \
03499          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03500          VALGRIND_RESTORE_STACK                                   \
03501          : /*out*/   "=r" (_res)                                  \
03502          : /*in*/    "r" (&_argvec[2])                            \
03503          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03504       );                                                          \
03505       lval = (__typeof__(lval)) _res;                             \
03506    } while (0)
03507 
03508 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
03509                                  arg7)                            \
03510    do {                                                           \
03511       volatile OrigFn        _orig = (orig);                      \
03512       volatile unsigned long _argvec[3+7];                        \
03513       volatile unsigned long _res;                                \
03514       /* _argvec[0] holds current r2 across the call */           \
03515       _argvec[1]   = (unsigned long)_orig.r2;                     \
03516       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03517       _argvec[2+1] = (unsigned long)arg1;                         \
03518       _argvec[2+2] = (unsigned long)arg2;                         \
03519       _argvec[2+3] = (unsigned long)arg3;                         \
03520       _argvec[2+4] = (unsigned long)arg4;                         \
03521       _argvec[2+5] = (unsigned long)arg5;                         \
03522       _argvec[2+6] = (unsigned long)arg6;                         \
03523       _argvec[2+7] = (unsigned long)arg7;                         \
03524       __asm__ volatile(                                           \
03525          VALGRIND_ALIGN_STACK                                     \
03526          "mr 12,%1\n\t"                                           \
03527          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03528          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03529          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03530          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03531          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03532          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03533          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03534          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03535          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
03536          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03537          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03538          "mr 12,%1\n\t"                                           \
03539          "mr %0,3\n\t"                                            \
03540          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03541          VALGRIND_RESTORE_STACK                                   \
03542          : /*out*/   "=r" (_res)                                  \
03543          : /*in*/    "r" (&_argvec[2])                            \
03544          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03545       );                                                          \
03546       lval = (__typeof__(lval)) _res;                             \
03547    } while (0)
03548 
03549 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
03550                                  arg7,arg8)                       \
03551    do {                                                           \
03552       volatile OrigFn        _orig = (orig);                      \
03553       volatile unsigned long _argvec[3+8];                        \
03554       volatile unsigned long _res;                                \
03555       /* _argvec[0] holds current r2 across the call */           \
03556       _argvec[1]   = (unsigned long)_orig.r2;                     \
03557       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03558       _argvec[2+1] = (unsigned long)arg1;                         \
03559       _argvec[2+2] = (unsigned long)arg2;                         \
03560       _argvec[2+3] = (unsigned long)arg3;                         \
03561       _argvec[2+4] = (unsigned long)arg4;                         \
03562       _argvec[2+5] = (unsigned long)arg5;                         \
03563       _argvec[2+6] = (unsigned long)arg6;                         \
03564       _argvec[2+7] = (unsigned long)arg7;                         \
03565       _argvec[2+8] = (unsigned long)arg8;                         \
03566       __asm__ volatile(                                           \
03567          VALGRIND_ALIGN_STACK                                     \
03568          "mr 12,%1\n\t"                                           \
03569          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03570          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03571          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03572          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03573          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03574          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03575          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03576          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03577          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
03578          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
03579          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03580          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03581          "mr 12,%1\n\t"                                           \
03582          "mr %0,3\n\t"                                            \
03583          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03584          VALGRIND_RESTORE_STACK                                   \
03585          : /*out*/   "=r" (_res)                                  \
03586          : /*in*/    "r" (&_argvec[2])                            \
03587          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03588       );                                                          \
03589       lval = (__typeof__(lval)) _res;                             \
03590    } while (0)
03591 
03592 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
03593                                  arg7,arg8,arg9)                  \
03594    do {                                                           \
03595       volatile OrigFn        _orig = (orig);                      \
03596       volatile unsigned long _argvec[3+9];                        \
03597       volatile unsigned long _res;                                \
03598       /* _argvec[0] holds current r2 across the call */           \
03599       _argvec[1]   = (unsigned long)_orig.r2;                     \
03600       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03601       _argvec[2+1] = (unsigned long)arg1;                         \
03602       _argvec[2+2] = (unsigned long)arg2;                         \
03603       _argvec[2+3] = (unsigned long)arg3;                         \
03604       _argvec[2+4] = (unsigned long)arg4;                         \
03605       _argvec[2+5] = (unsigned long)arg5;                         \
03606       _argvec[2+6] = (unsigned long)arg6;                         \
03607       _argvec[2+7] = (unsigned long)arg7;                         \
03608       _argvec[2+8] = (unsigned long)arg8;                         \
03609       _argvec[2+9] = (unsigned long)arg9;                         \
03610       __asm__ volatile(                                           \
03611          VALGRIND_ALIGN_STACK                                     \
03612          "mr 12,%1\n\t"                                           \
03613          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03614          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03615          "addi 1,1,-128\n\t"  /* expand stack frame */            \
03616          /* arg9 */                                               \
03617          "ld  3,72(12)\n\t"                                       \
03618          "std 3,96(1)\n\t"                                        \
03619          /* args1-8 */                                            \
03620          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03621          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03622          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03623          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03624          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03625          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03626          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
03627          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
03628          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03629          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03630          "mr 12,%1\n\t"                                           \
03631          "mr %0,3\n\t"                                            \
03632          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03633          VALGRIND_RESTORE_STACK                                   \
03634          : /*out*/   "=r" (_res)                                  \
03635          : /*in*/    "r" (&_argvec[2])                            \
03636          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03637       );                                                          \
03638       lval = (__typeof__(lval)) _res;                             \
03639    } while (0)
03640 
03641 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
03642                                   arg7,arg8,arg9,arg10)           \
03643    do {                                                           \
03644       volatile OrigFn        _orig = (orig);                      \
03645       volatile unsigned long _argvec[3+10];                       \
03646       volatile unsigned long _res;                                \
03647       /* _argvec[0] holds current r2 across the call */           \
03648       _argvec[1]   = (unsigned long)_orig.r2;                     \
03649       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03650       _argvec[2+1] = (unsigned long)arg1;                         \
03651       _argvec[2+2] = (unsigned long)arg2;                         \
03652       _argvec[2+3] = (unsigned long)arg3;                         \
03653       _argvec[2+4] = (unsigned long)arg4;                         \
03654       _argvec[2+5] = (unsigned long)arg5;                         \
03655       _argvec[2+6] = (unsigned long)arg6;                         \
03656       _argvec[2+7] = (unsigned long)arg7;                         \
03657       _argvec[2+8] = (unsigned long)arg8;                         \
03658       _argvec[2+9] = (unsigned long)arg9;                         \
03659       _argvec[2+10] = (unsigned long)arg10;                       \
03660       __asm__ volatile(                                           \
03661          VALGRIND_ALIGN_STACK                                     \
03662          "mr 12,%1\n\t"                                           \
03663          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03664          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03665          "addi 1,1,-128\n\t"  /* expand stack frame */            \
03666          /* arg10 */                                              \
03667          "ld  3,80(12)\n\t"                                       \
03668          "std 3,104(1)\n\t"                                       \
03669          /* arg9 */                                               \
03670          "ld  3,72(12)\n\t"                                       \
03671          "std 3,96(1)\n\t"                                        \
03672          /* args1-8 */                                            \
03673          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03674          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03675          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03676          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03677          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03678          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03679          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
03680          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
03681          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03682          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03683          "mr 12,%1\n\t"                                           \
03684          "mr %0,3\n\t"                                            \
03685          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03686          VALGRIND_RESTORE_STACK                                   \
03687          : /*out*/   "=r" (_res)                                  \
03688          : /*in*/    "r" (&_argvec[2])                            \
03689          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03690       );                                                          \
03691       lval = (__typeof__(lval)) _res;                             \
03692    } while (0)
03693 
03694 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
03695                                   arg7,arg8,arg9,arg10,arg11)     \
03696    do {                                                           \
03697       volatile OrigFn        _orig = (orig);                      \
03698       volatile unsigned long _argvec[3+11];                       \
03699       volatile unsigned long _res;                                \
03700       /* _argvec[0] holds current r2 across the call */           \
03701       _argvec[1]   = (unsigned long)_orig.r2;                     \
03702       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03703       _argvec[2+1] = (unsigned long)arg1;                         \
03704       _argvec[2+2] = (unsigned long)arg2;                         \
03705       _argvec[2+3] = (unsigned long)arg3;                         \
03706       _argvec[2+4] = (unsigned long)arg4;                         \
03707       _argvec[2+5] = (unsigned long)arg5;                         \
03708       _argvec[2+6] = (unsigned long)arg6;                         \
03709       _argvec[2+7] = (unsigned long)arg7;                         \
03710       _argvec[2+8] = (unsigned long)arg8;                         \
03711       _argvec[2+9] = (unsigned long)arg9;                         \
03712       _argvec[2+10] = (unsigned long)arg10;                       \
03713       _argvec[2+11] = (unsigned long)arg11;                       \
03714       __asm__ volatile(                                           \
03715          VALGRIND_ALIGN_STACK                                     \
03716          "mr 12,%1\n\t"                                           \
03717          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03718          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03719          "addi 1,1,-144\n\t"  /* expand stack frame */            \
03720          /* arg11 */                                              \
03721          "ld  3,88(12)\n\t"                                       \
03722          "std 3,112(1)\n\t"                                       \
03723          /* arg10 */                                              \
03724          "ld  3,80(12)\n\t"                                       \
03725          "std 3,104(1)\n\t"                                       \
03726          /* arg9 */                                               \
03727          "ld  3,72(12)\n\t"                                       \
03728          "std 3,96(1)\n\t"                                        \
03729          /* args1-8 */                                            \
03730          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03731          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03732          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03733          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03734          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03735          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03736          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
03737          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
03738          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03739          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03740          "mr 12,%1\n\t"                                           \
03741          "mr %0,3\n\t"                                            \
03742          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03743          VALGRIND_RESTORE_STACK                                   \
03744          : /*out*/   "=r" (_res)                                  \
03745          : /*in*/    "r" (&_argvec[2])                            \
03746          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03747       );                                                          \
03748       lval = (__typeof__(lval)) _res;                             \
03749    } while (0)
03750 
03751 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
03752                                 arg7,arg8,arg9,arg10,arg11,arg12) \
03753    do {                                                           \
03754       volatile OrigFn        _orig = (orig);                      \
03755       volatile unsigned long _argvec[3+12];                       \
03756       volatile unsigned long _res;                                \
03757       /* _argvec[0] holds current r2 across the call */           \
03758       _argvec[1]   = (unsigned long)_orig.r2;                     \
03759       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
03760       _argvec[2+1] = (unsigned long)arg1;                         \
03761       _argvec[2+2] = (unsigned long)arg2;                         \
03762       _argvec[2+3] = (unsigned long)arg3;                         \
03763       _argvec[2+4] = (unsigned long)arg4;                         \
03764       _argvec[2+5] = (unsigned long)arg5;                         \
03765       _argvec[2+6] = (unsigned long)arg6;                         \
03766       _argvec[2+7] = (unsigned long)arg7;                         \
03767       _argvec[2+8] = (unsigned long)arg8;                         \
03768       _argvec[2+9] = (unsigned long)arg9;                         \
03769       _argvec[2+10] = (unsigned long)arg10;                       \
03770       _argvec[2+11] = (unsigned long)arg11;                       \
03771       _argvec[2+12] = (unsigned long)arg12;                       \
03772       __asm__ volatile(                                           \
03773          VALGRIND_ALIGN_STACK                                     \
03774          "mr 12,%1\n\t"                                           \
03775          "std 2,-16(12)\n\t"  /* save tocptr */                   \
03776          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
03777          "addi 1,1,-144\n\t"  /* expand stack frame */            \
03778          /* arg12 */                                              \
03779          "ld  3,96(12)\n\t"                                       \
03780          "std 3,120(1)\n\t"                                       \
03781          /* arg11 */                                              \
03782          "ld  3,88(12)\n\t"                                       \
03783          "std 3,112(1)\n\t"                                       \
03784          /* arg10 */                                              \
03785          "ld  3,80(12)\n\t"                                       \
03786          "std 3,104(1)\n\t"                                       \
03787          /* arg9 */                                               \
03788          "ld  3,72(12)\n\t"                                       \
03789          "std 3,96(1)\n\t"                                        \
03790          /* args1-8 */                                            \
03791          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
03792          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
03793          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
03794          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
03795          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
03796          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
03797          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
03798          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
03799          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
03800          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
03801          "mr 12,%1\n\t"                                           \
03802          "mr %0,3\n\t"                                            \
03803          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
03804          VALGRIND_RESTORE_STACK                                   \
03805          : /*out*/   "=r" (_res)                                  \
03806          : /*in*/    "r" (&_argvec[2])                            \
03807          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
03808       );                                                          \
03809       lval = (__typeof__(lval)) _res;                             \
03810    } while (0)
03811 
03812 #endif /* PLAT_ppc64le_linux */
03813 
03814 /* ------------------------- arm-linux ------------------------- */
03815 
03816 #if defined(PLAT_arm_linux)
03817 
03818 /* These regs are trashed by the hidden call. */
03819 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4", "r12", "r14"
03820 
03821 /* Macros to save and align the stack before making a function
03822    call and restore it afterwards as gcc may not keep the stack
03823    pointer aligned if it doesn't realise calls are being made
03824    to other functions. */
03825 
03826 /* This is a bit tricky.  We store the original stack pointer in r10
03827    as it is callee-saves.  gcc doesn't allow the use of r11 for some
03828    reason.  Also, we can't directly "bic" the stack pointer in thumb
03829    mode since r13 isn't an allowed register number in that context.
03830    So use r4 as a temporary, since that is about to get trashed
03831    anyway, just after each use of this macro.  Side effect is we need
03832    to be very careful about any future changes, since
03833    VALGRIND_ALIGN_STACK simply assumes r4 is usable. */
03834 #define VALGRIND_ALIGN_STACK               \
03835       "mov r10, sp\n\t"                    \
03836       "mov r4,  sp\n\t"                    \
03837       "bic r4,  r4, #7\n\t"                \
03838       "mov sp,  r4\n\t"
03839 #define VALGRIND_RESTORE_STACK             \
03840       "mov sp,  r10\n\t"
03841 
03842 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
03843    long) == 4. */
03844 
03845 #define CALL_FN_W_v(lval, orig)                                   \
03846    do {                                                           \
03847       volatile OrigFn        _orig = (orig);                      \
03848       volatile unsigned long _argvec[1];                          \
03849       volatile unsigned long _res;                                \
03850       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03851       __asm__ volatile(                                           \
03852          VALGRIND_ALIGN_STACK                                     \
03853          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
03854          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
03855          VALGRIND_RESTORE_STACK                                   \
03856          "mov %0, r0\n"                                           \
03857          : /*out*/   "=r" (_res)                                  \
03858          : /*in*/    "0" (&_argvec[0])                            \
03859          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
03860       );                                                          \
03861       lval = (__typeof__(lval)) _res;                             \
03862    } while (0)
03863 
03864 #define CALL_FN_W_W(lval, orig, arg1)                             \
03865    do {                                                           \
03866       volatile OrigFn        _orig = (orig);                      \
03867       volatile unsigned long _argvec[2];                          \
03868       volatile unsigned long _res;                                \
03869       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03870       _argvec[1] = (unsigned long)(arg1);                         \
03871       __asm__ volatile(                                           \
03872          VALGRIND_ALIGN_STACK                                     \
03873          "ldr r0, [%1, #4] \n\t"                                  \
03874          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
03875          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
03876          VALGRIND_RESTORE_STACK                                   \
03877          "mov %0, r0\n"                                           \
03878          : /*out*/   "=r" (_res)                                  \
03879          : /*in*/    "0" (&_argvec[0])                            \
03880          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
03881       );                                                          \
03882       lval = (__typeof__(lval)) _res;                             \
03883    } while (0)
03884 
03885 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
03886    do {                                                           \
03887       volatile OrigFn        _orig = (orig);                      \
03888       volatile unsigned long _argvec[3];                          \
03889       volatile unsigned long _res;                                \
03890       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03891       _argvec[1] = (unsigned long)(arg1);                         \
03892       _argvec[2] = (unsigned long)(arg2);                         \
03893       __asm__ volatile(                                           \
03894          VALGRIND_ALIGN_STACK                                     \
03895          "ldr r0, [%1, #4] \n\t"                                  \
03896          "ldr r1, [%1, #8] \n\t"                                  \
03897          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
03898          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
03899          VALGRIND_RESTORE_STACK                                   \
03900          "mov %0, r0\n"                                           \
03901          : /*out*/   "=r" (_res)                                  \
03902          : /*in*/    "0" (&_argvec[0])                            \
03903          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
03904       );                                                          \
03905       lval = (__typeof__(lval)) _res;                             \
03906    } while (0)
03907 
03908 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
03909    do {                                                           \
03910       volatile OrigFn        _orig = (orig);                      \
03911       volatile unsigned long _argvec[4];                          \
03912       volatile unsigned long _res;                                \
03913       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03914       _argvec[1] = (unsigned long)(arg1);                         \
03915       _argvec[2] = (unsigned long)(arg2);                         \
03916       _argvec[3] = (unsigned long)(arg3);                         \
03917       __asm__ volatile(                                           \
03918          VALGRIND_ALIGN_STACK                                     \
03919          "ldr r0, [%1, #4] \n\t"                                  \
03920          "ldr r1, [%1, #8] \n\t"                                  \
03921          "ldr r2, [%1, #12] \n\t"                                 \
03922          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
03923          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
03924          VALGRIND_RESTORE_STACK                                   \
03925          "mov %0, r0\n"                                           \
03926          : /*out*/   "=r" (_res)                                  \
03927          : /*in*/    "0" (&_argvec[0])                            \
03928          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
03929       );                                                          \
03930       lval = (__typeof__(lval)) _res;                             \
03931    } while (0)
03932 
03933 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
03934    do {                                                           \
03935       volatile OrigFn        _orig = (orig);                      \
03936       volatile unsigned long _argvec[5];                          \
03937       volatile unsigned long _res;                                \
03938       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03939       _argvec[1] = (unsigned long)(arg1);                         \
03940       _argvec[2] = (unsigned long)(arg2);                         \
03941       _argvec[3] = (unsigned long)(arg3);                         \
03942       _argvec[4] = (unsigned long)(arg4);                         \
03943       __asm__ volatile(                                           \
03944          VALGRIND_ALIGN_STACK                                     \
03945          "ldr r0, [%1, #4] \n\t"                                  \
03946          "ldr r1, [%1, #8] \n\t"                                  \
03947          "ldr r2, [%1, #12] \n\t"                                 \
03948          "ldr r3, [%1, #16] \n\t"                                 \
03949          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
03950          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
03951          VALGRIND_RESTORE_STACK                                   \
03952          "mov %0, r0"                                             \
03953          : /*out*/   "=r" (_res)                                  \
03954          : /*in*/    "0" (&_argvec[0])                            \
03955          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
03956       );                                                          \
03957       lval = (__typeof__(lval)) _res;                             \
03958    } while (0)
03959 
03960 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
03961    do {                                                           \
03962       volatile OrigFn        _orig = (orig);                      \
03963       volatile unsigned long _argvec[6];                          \
03964       volatile unsigned long _res;                                \
03965       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03966       _argvec[1] = (unsigned long)(arg1);                         \
03967       _argvec[2] = (unsigned long)(arg2);                         \
03968       _argvec[3] = (unsigned long)(arg3);                         \
03969       _argvec[4] = (unsigned long)(arg4);                         \
03970       _argvec[5] = (unsigned long)(arg5);                         \
03971       __asm__ volatile(                                           \
03972          VALGRIND_ALIGN_STACK                                     \
03973          "sub sp, sp, #4 \n\t"                                    \
03974          "ldr r0, [%1, #20] \n\t"                                 \
03975          "push {r0} \n\t"                                         \
03976          "ldr r0, [%1, #4] \n\t"                                  \
03977          "ldr r1, [%1, #8] \n\t"                                  \
03978          "ldr r2, [%1, #12] \n\t"                                 \
03979          "ldr r3, [%1, #16] \n\t"                                 \
03980          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
03981          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
03982          VALGRIND_RESTORE_STACK                                   \
03983          "mov %0, r0"                                             \
03984          : /*out*/   "=r" (_res)                                  \
03985          : /*in*/    "0" (&_argvec[0])                            \
03986          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
03987       );                                                          \
03988       lval = (__typeof__(lval)) _res;                             \
03989    } while (0)
03990 
03991 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
03992    do {                                                           \
03993       volatile OrigFn        _orig = (orig);                      \
03994       volatile unsigned long _argvec[7];                          \
03995       volatile unsigned long _res;                                \
03996       _argvec[0] = (unsigned long)_orig.nraddr;                   \
03997       _argvec[1] = (unsigned long)(arg1);                         \
03998       _argvec[2] = (unsigned long)(arg2);                         \
03999       _argvec[3] = (unsigned long)(arg3);                         \
04000       _argvec[4] = (unsigned long)(arg4);                         \
04001       _argvec[5] = (unsigned long)(arg5);                         \
04002       _argvec[6] = (unsigned long)(arg6);                         \
04003       __asm__ volatile(                                           \
04004          VALGRIND_ALIGN_STACK                                     \
04005          "ldr r0, [%1, #20] \n\t"                                 \
04006          "ldr r1, [%1, #24] \n\t"                                 \
04007          "push {r0, r1} \n\t"                                     \
04008          "ldr r0, [%1, #4] \n\t"                                  \
04009          "ldr r1, [%1, #8] \n\t"                                  \
04010          "ldr r2, [%1, #12] \n\t"                                 \
04011          "ldr r3, [%1, #16] \n\t"                                 \
04012          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04013          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04014          VALGRIND_RESTORE_STACK                                   \
04015          "mov %0, r0"                                             \
04016          : /*out*/   "=r" (_res)                                  \
04017          : /*in*/    "0" (&_argvec[0])                            \
04018          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04019       );                                                          \
04020       lval = (__typeof__(lval)) _res;                             \
04021    } while (0)
04022 
04023 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
04024                                  arg7)                            \
04025    do {                                                           \
04026       volatile OrigFn        _orig = (orig);                      \
04027       volatile unsigned long _argvec[8];                          \
04028       volatile unsigned long _res;                                \
04029       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04030       _argvec[1] = (unsigned long)(arg1);                         \
04031       _argvec[2] = (unsigned long)(arg2);                         \
04032       _argvec[3] = (unsigned long)(arg3);                         \
04033       _argvec[4] = (unsigned long)(arg4);                         \
04034       _argvec[5] = (unsigned long)(arg5);                         \
04035       _argvec[6] = (unsigned long)(arg6);                         \
04036       _argvec[7] = (unsigned long)(arg7);                         \
04037       __asm__ volatile(                                           \
04038          VALGRIND_ALIGN_STACK                                     \
04039          "sub sp, sp, #4 \n\t"                                    \
04040          "ldr r0, [%1, #20] \n\t"                                 \
04041          "ldr r1, [%1, #24] \n\t"                                 \
04042          "ldr r2, [%1, #28] \n\t"                                 \
04043          "push {r0, r1, r2} \n\t"                                 \
04044          "ldr r0, [%1, #4] \n\t"                                  \
04045          "ldr r1, [%1, #8] \n\t"                                  \
04046          "ldr r2, [%1, #12] \n\t"                                 \
04047          "ldr r3, [%1, #16] \n\t"                                 \
04048          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04049          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04050          VALGRIND_RESTORE_STACK                                   \
04051          "mov %0, r0"                                             \
04052          : /*out*/   "=r" (_res)                                  \
04053          : /*in*/    "0" (&_argvec[0])                            \
04054          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04055       );                                                          \
04056       lval = (__typeof__(lval)) _res;                             \
04057    } while (0)
04058 
04059 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
04060                                  arg7,arg8)                       \
04061    do {                                                           \
04062       volatile OrigFn        _orig = (orig);                      \
04063       volatile unsigned long _argvec[9];                          \
04064       volatile unsigned long _res;                                \
04065       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04066       _argvec[1] = (unsigned long)(arg1);                         \
04067       _argvec[2] = (unsigned long)(arg2);                         \
04068       _argvec[3] = (unsigned long)(arg3);                         \
04069       _argvec[4] = (unsigned long)(arg4);                         \
04070       _argvec[5] = (unsigned long)(arg5);                         \
04071       _argvec[6] = (unsigned long)(arg6);                         \
04072       _argvec[7] = (unsigned long)(arg7);                         \
04073       _argvec[8] = (unsigned long)(arg8);                         \
04074       __asm__ volatile(                                           \
04075          VALGRIND_ALIGN_STACK                                     \
04076          "ldr r0, [%1, #20] \n\t"                                 \
04077          "ldr r1, [%1, #24] \n\t"                                 \
04078          "ldr r2, [%1, #28] \n\t"                                 \
04079          "ldr r3, [%1, #32] \n\t"                                 \
04080          "push {r0, r1, r2, r3} \n\t"                             \
04081          "ldr r0, [%1, #4] \n\t"                                  \
04082          "ldr r1, [%1, #8] \n\t"                                  \
04083          "ldr r2, [%1, #12] \n\t"                                 \
04084          "ldr r3, [%1, #16] \n\t"                                 \
04085          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04086          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04087          VALGRIND_RESTORE_STACK                                   \
04088          "mov %0, r0"                                             \
04089          : /*out*/   "=r" (_res)                                  \
04090          : /*in*/    "0" (&_argvec[0])                            \
04091          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04092       );                                                          \
04093       lval = (__typeof__(lval)) _res;                             \
04094    } while (0)
04095 
04096 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
04097                                  arg7,arg8,arg9)                  \
04098    do {                                                           \
04099       volatile OrigFn        _orig = (orig);                      \
04100       volatile unsigned long _argvec[10];                         \
04101       volatile unsigned long _res;                                \
04102       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04103       _argvec[1] = (unsigned long)(arg1);                         \
04104       _argvec[2] = (unsigned long)(arg2);                         \
04105       _argvec[3] = (unsigned long)(arg3);                         \
04106       _argvec[4] = (unsigned long)(arg4);                         \
04107       _argvec[5] = (unsigned long)(arg5);                         \
04108       _argvec[6] = (unsigned long)(arg6);                         \
04109       _argvec[7] = (unsigned long)(arg7);                         \
04110       _argvec[8] = (unsigned long)(arg8);                         \
04111       _argvec[9] = (unsigned long)(arg9);                         \
04112       __asm__ volatile(                                           \
04113          VALGRIND_ALIGN_STACK                                     \
04114          "sub sp, sp, #4 \n\t"                                    \
04115          "ldr r0, [%1, #20] \n\t"                                 \
04116          "ldr r1, [%1, #24] \n\t"                                 \
04117          "ldr r2, [%1, #28] \n\t"                                 \
04118          "ldr r3, [%1, #32] \n\t"                                 \
04119          "ldr r4, [%1, #36] \n\t"                                 \
04120          "push {r0, r1, r2, r3, r4} \n\t"                         \
04121          "ldr r0, [%1, #4] \n\t"                                  \
04122          "ldr r1, [%1, #8] \n\t"                                  \
04123          "ldr r2, [%1, #12] \n\t"                                 \
04124          "ldr r3, [%1, #16] \n\t"                                 \
04125          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04126          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04127          VALGRIND_RESTORE_STACK                                   \
04128          "mov %0, r0"                                             \
04129          : /*out*/   "=r" (_res)                                  \
04130          : /*in*/    "0" (&_argvec[0])                            \
04131          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04132       );                                                          \
04133       lval = (__typeof__(lval)) _res;                             \
04134    } while (0)
04135 
04136 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
04137                                   arg7,arg8,arg9,arg10)           \
04138    do {                                                           \
04139       volatile OrigFn        _orig = (orig);                      \
04140       volatile unsigned long _argvec[11];                         \
04141       volatile unsigned long _res;                                \
04142       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04143       _argvec[1] = (unsigned long)(arg1);                         \
04144       _argvec[2] = (unsigned long)(arg2);                         \
04145       _argvec[3] = (unsigned long)(arg3);                         \
04146       _argvec[4] = (unsigned long)(arg4);                         \
04147       _argvec[5] = (unsigned long)(arg5);                         \
04148       _argvec[6] = (unsigned long)(arg6);                         \
04149       _argvec[7] = (unsigned long)(arg7);                         \
04150       _argvec[8] = (unsigned long)(arg8);                         \
04151       _argvec[9] = (unsigned long)(arg9);                         \
04152       _argvec[10] = (unsigned long)(arg10);                       \
04153       __asm__ volatile(                                           \
04154          VALGRIND_ALIGN_STACK                                     \
04155          "ldr r0, [%1, #40] \n\t"                                 \
04156          "push {r0} \n\t"                                         \
04157          "ldr r0, [%1, #20] \n\t"                                 \
04158          "ldr r1, [%1, #24] \n\t"                                 \
04159          "ldr r2, [%1, #28] \n\t"                                 \
04160          "ldr r3, [%1, #32] \n\t"                                 \
04161          "ldr r4, [%1, #36] \n\t"                                 \
04162          "push {r0, r1, r2, r3, r4} \n\t"                         \
04163          "ldr r0, [%1, #4] \n\t"                                  \
04164          "ldr r1, [%1, #8] \n\t"                                  \
04165          "ldr r2, [%1, #12] \n\t"                                 \
04166          "ldr r3, [%1, #16] \n\t"                                 \
04167          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04168          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04169          VALGRIND_RESTORE_STACK                                   \
04170          "mov %0, r0"                                             \
04171          : /*out*/   "=r" (_res)                                  \
04172          : /*in*/    "0" (&_argvec[0])                            \
04173          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04174       );                                                          \
04175       lval = (__typeof__(lval)) _res;                             \
04176    } while (0)
04177 
04178 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
04179                                   arg6,arg7,arg8,arg9,arg10,      \
04180                                   arg11)                          \
04181    do {                                                           \
04182       volatile OrigFn        _orig = (orig);                      \
04183       volatile unsigned long _argvec[12];                         \
04184       volatile unsigned long _res;                                \
04185       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04186       _argvec[1] = (unsigned long)(arg1);                         \
04187       _argvec[2] = (unsigned long)(arg2);                         \
04188       _argvec[3] = (unsigned long)(arg3);                         \
04189       _argvec[4] = (unsigned long)(arg4);                         \
04190       _argvec[5] = (unsigned long)(arg5);                         \
04191       _argvec[6] = (unsigned long)(arg6);                         \
04192       _argvec[7] = (unsigned long)(arg7);                         \
04193       _argvec[8] = (unsigned long)(arg8);                         \
04194       _argvec[9] = (unsigned long)(arg9);                         \
04195       _argvec[10] = (unsigned long)(arg10);                       \
04196       _argvec[11] = (unsigned long)(arg11);                       \
04197       __asm__ volatile(                                           \
04198          VALGRIND_ALIGN_STACK                                     \
04199          "sub sp, sp, #4 \n\t"                                    \
04200          "ldr r0, [%1, #40] \n\t"                                 \
04201          "ldr r1, [%1, #44] \n\t"                                 \
04202          "push {r0, r1} \n\t"                                     \
04203          "ldr r0, [%1, #20] \n\t"                                 \
04204          "ldr r1, [%1, #24] \n\t"                                 \
04205          "ldr r2, [%1, #28] \n\t"                                 \
04206          "ldr r3, [%1, #32] \n\t"                                 \
04207          "ldr r4, [%1, #36] \n\t"                                 \
04208          "push {r0, r1, r2, r3, r4} \n\t"                         \
04209          "ldr r0, [%1, #4] \n\t"                                  \
04210          "ldr r1, [%1, #8] \n\t"                                  \
04211          "ldr r2, [%1, #12] \n\t"                                 \
04212          "ldr r3, [%1, #16] \n\t"                                 \
04213          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04214          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04215          VALGRIND_RESTORE_STACK                                   \
04216          "mov %0, r0"                                             \
04217          : /*out*/   "=r" (_res)                                  \
04218          : /*in*/    "0" (&_argvec[0])                            \
04219          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04220       );                                                          \
04221       lval = (__typeof__(lval)) _res;                             \
04222    } while (0)
04223 
04224 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
04225                                   arg6,arg7,arg8,arg9,arg10,      \
04226                                   arg11,arg12)                    \
04227    do {                                                           \
04228       volatile OrigFn        _orig = (orig);                      \
04229       volatile unsigned long _argvec[13];                         \
04230       volatile unsigned long _res;                                \
04231       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04232       _argvec[1] = (unsigned long)(arg1);                         \
04233       _argvec[2] = (unsigned long)(arg2);                         \
04234       _argvec[3] = (unsigned long)(arg3);                         \
04235       _argvec[4] = (unsigned long)(arg4);                         \
04236       _argvec[5] = (unsigned long)(arg5);                         \
04237       _argvec[6] = (unsigned long)(arg6);                         \
04238       _argvec[7] = (unsigned long)(arg7);                         \
04239       _argvec[8] = (unsigned long)(arg8);                         \
04240       _argvec[9] = (unsigned long)(arg9);                         \
04241       _argvec[10] = (unsigned long)(arg10);                       \
04242       _argvec[11] = (unsigned long)(arg11);                       \
04243       _argvec[12] = (unsigned long)(arg12);                       \
04244       __asm__ volatile(                                           \
04245          VALGRIND_ALIGN_STACK                                     \
04246          "ldr r0, [%1, #40] \n\t"                                 \
04247          "ldr r1, [%1, #44] \n\t"                                 \
04248          "ldr r2, [%1, #48] \n\t"                                 \
04249          "push {r0, r1, r2} \n\t"                                 \
04250          "ldr r0, [%1, #20] \n\t"                                 \
04251          "ldr r1, [%1, #24] \n\t"                                 \
04252          "ldr r2, [%1, #28] \n\t"                                 \
04253          "ldr r3, [%1, #32] \n\t"                                 \
04254          "ldr r4, [%1, #36] \n\t"                                 \
04255          "push {r0, r1, r2, r3, r4} \n\t"                         \
04256          "ldr r0, [%1, #4] \n\t"                                  \
04257          "ldr r1, [%1, #8] \n\t"                                  \
04258          "ldr r2, [%1, #12] \n\t"                                 \
04259          "ldr r3, [%1, #16] \n\t"                                 \
04260          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
04261          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
04262          VALGRIND_RESTORE_STACK                                   \
04263          "mov %0, r0"                                             \
04264          : /*out*/   "=r" (_res)                                  \
04265          : /*in*/    "0" (&_argvec[0])                            \
04266          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
04267       );                                                          \
04268       lval = (__typeof__(lval)) _res;                             \
04269    } while (0)
04270 
04271 #endif /* PLAT_arm_linux */
04272 
04273 /* ------------------------ arm64-linux ------------------------ */
04274 
04275 #if defined(PLAT_arm64_linux)
04276 
04277 /* These regs are trashed by the hidden call. */
04278 #define __CALLER_SAVED_REGS \
04279      "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9",   \
04280      "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17",      \
04281      "x18", "x19", "x20", "x30",                                  \
04282      "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",  \
04283      "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",      \
04284      "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",      \
04285      "v26", "v27", "v28", "v29", "v30", "v31"
04286 
04287 /* x21 is callee-saved, so we can use it to save and restore SP around
04288    the hidden call. */
04289 #define VALGRIND_ALIGN_STACK               \
04290       "mov x21, sp\n\t"                    \
04291       "bic sp, x21, #15\n\t"
04292 #define VALGRIND_RESTORE_STACK             \
04293       "mov sp,  x21\n\t"
04294 
04295 /* These CALL_FN_ macros assume that on arm64-linux,
04296    sizeof(unsigned long) == 8. */
04297 
04298 #define CALL_FN_W_v(lval, orig)                                   \
04299    do {                                                           \
04300       volatile OrigFn        _orig = (orig);                      \
04301       volatile unsigned long _argvec[1];                          \
04302       volatile unsigned long _res;                                \
04303       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04304       __asm__ volatile(                                           \
04305          VALGRIND_ALIGN_STACK                                     \
04306          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04307          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04308          VALGRIND_RESTORE_STACK                                   \
04309          "mov %0, x0\n"                                           \
04310          : /*out*/   "=r" (_res)                                  \
04311          : /*in*/    "0" (&_argvec[0])                            \
04312          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04313       );                                                          \
04314       lval = (__typeof__(lval)) _res;                             \
04315    } while (0)
04316 
04317 #define CALL_FN_W_W(lval, orig, arg1)                             \
04318    do {                                                           \
04319       volatile OrigFn        _orig = (orig);                      \
04320       volatile unsigned long _argvec[2];                          \
04321       volatile unsigned long _res;                                \
04322       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04323       _argvec[1] = (unsigned long)(arg1);                         \
04324       __asm__ volatile(                                           \
04325          VALGRIND_ALIGN_STACK                                     \
04326          "ldr x0, [%1, #8] \n\t"                                  \
04327          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04328          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04329          VALGRIND_RESTORE_STACK                                   \
04330          "mov %0, x0\n"                                           \
04331          : /*out*/   "=r" (_res)                                  \
04332          : /*in*/    "0" (&_argvec[0])                            \
04333          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04334       );                                                          \
04335       lval = (__typeof__(lval)) _res;                             \
04336    } while (0)
04337 
04338 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
04339    do {                                                           \
04340       volatile OrigFn        _orig = (orig);                      \
04341       volatile unsigned long _argvec[3];                          \
04342       volatile unsigned long _res;                                \
04343       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04344       _argvec[1] = (unsigned long)(arg1);                         \
04345       _argvec[2] = (unsigned long)(arg2);                         \
04346       __asm__ volatile(                                           \
04347          VALGRIND_ALIGN_STACK                                     \
04348          "ldr x0, [%1, #8] \n\t"                                  \
04349          "ldr x1, [%1, #16] \n\t"                                 \
04350          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04351          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04352          VALGRIND_RESTORE_STACK                                   \
04353          "mov %0, x0\n"                                           \
04354          : /*out*/   "=r" (_res)                                  \
04355          : /*in*/    "0" (&_argvec[0])                            \
04356          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04357       );                                                          \
04358       lval = (__typeof__(lval)) _res;                             \
04359    } while (0)
04360 
04361 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
04362    do {                                                           \
04363       volatile OrigFn        _orig = (orig);                      \
04364       volatile unsigned long _argvec[4];                          \
04365       volatile unsigned long _res;                                \
04366       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04367       _argvec[1] = (unsigned long)(arg1);                         \
04368       _argvec[2] = (unsigned long)(arg2);                         \
04369       _argvec[3] = (unsigned long)(arg3);                         \
04370       __asm__ volatile(                                           \
04371          VALGRIND_ALIGN_STACK                                     \
04372          "ldr x0, [%1, #8] \n\t"                                  \
04373          "ldr x1, [%1, #16] \n\t"                                 \
04374          "ldr x2, [%1, #24] \n\t"                                 \
04375          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04376          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04377          VALGRIND_RESTORE_STACK                                   \
04378          "mov %0, x0\n"                                           \
04379          : /*out*/   "=r" (_res)                                  \
04380          : /*in*/    "0" (&_argvec[0])                            \
04381          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04382       );                                                          \
04383       lval = (__typeof__(lval)) _res;                             \
04384    } while (0)
04385 
04386 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
04387    do {                                                           \
04388       volatile OrigFn        _orig = (orig);                      \
04389       volatile unsigned long _argvec[5];                          \
04390       volatile unsigned long _res;                                \
04391       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04392       _argvec[1] = (unsigned long)(arg1);                         \
04393       _argvec[2] = (unsigned long)(arg2);                         \
04394       _argvec[3] = (unsigned long)(arg3);                         \
04395       _argvec[4] = (unsigned long)(arg4);                         \
04396       __asm__ volatile(                                           \
04397          VALGRIND_ALIGN_STACK                                     \
04398          "ldr x0, [%1, #8] \n\t"                                  \
04399          "ldr x1, [%1, #16] \n\t"                                 \
04400          "ldr x2, [%1, #24] \n\t"                                 \
04401          "ldr x3, [%1, #32] \n\t"                                 \
04402          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04403          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04404          VALGRIND_RESTORE_STACK                                   \
04405          "mov %0, x0"                                             \
04406          : /*out*/   "=r" (_res)                                  \
04407          : /*in*/    "0" (&_argvec[0])                            \
04408          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04409       );                                                          \
04410       lval = (__typeof__(lval)) _res;                             \
04411    } while (0)
04412 
04413 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
04414    do {                                                           \
04415       volatile OrigFn        _orig = (orig);                      \
04416       volatile unsigned long _argvec[6];                          \
04417       volatile unsigned long _res;                                \
04418       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04419       _argvec[1] = (unsigned long)(arg1);                         \
04420       _argvec[2] = (unsigned long)(arg2);                         \
04421       _argvec[3] = (unsigned long)(arg3);                         \
04422       _argvec[4] = (unsigned long)(arg4);                         \
04423       _argvec[5] = (unsigned long)(arg5);                         \
04424       __asm__ volatile(                                           \
04425          VALGRIND_ALIGN_STACK                                     \
04426          "ldr x0, [%1, #8] \n\t"                                  \
04427          "ldr x1, [%1, #16] \n\t"                                 \
04428          "ldr x2, [%1, #24] \n\t"                                 \
04429          "ldr x3, [%1, #32] \n\t"                                 \
04430          "ldr x4, [%1, #40] \n\t"                                 \
04431          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04432          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04433          VALGRIND_RESTORE_STACK                                   \
04434          "mov %0, x0"                                             \
04435          : /*out*/   "=r" (_res)                                  \
04436          : /*in*/    "0" (&_argvec[0])                            \
04437          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04438       );                                                          \
04439       lval = (__typeof__(lval)) _res;                             \
04440    } while (0)
04441 
04442 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
04443    do {                                                           \
04444       volatile OrigFn        _orig = (orig);                      \
04445       volatile unsigned long _argvec[7];                          \
04446       volatile unsigned long _res;                                \
04447       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04448       _argvec[1] = (unsigned long)(arg1);                         \
04449       _argvec[2] = (unsigned long)(arg2);                         \
04450       _argvec[3] = (unsigned long)(arg3);                         \
04451       _argvec[4] = (unsigned long)(arg4);                         \
04452       _argvec[5] = (unsigned long)(arg5);                         \
04453       _argvec[6] = (unsigned long)(arg6);                         \
04454       __asm__ volatile(                                           \
04455          VALGRIND_ALIGN_STACK                                     \
04456          "ldr x0, [%1, #8] \n\t"                                  \
04457          "ldr x1, [%1, #16] \n\t"                                 \
04458          "ldr x2, [%1, #24] \n\t"                                 \
04459          "ldr x3, [%1, #32] \n\t"                                 \
04460          "ldr x4, [%1, #40] \n\t"                                 \
04461          "ldr x5, [%1, #48] \n\t"                                 \
04462          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04463          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04464          VALGRIND_RESTORE_STACK                                   \
04465          "mov %0, x0"                                             \
04466          : /*out*/   "=r" (_res)                                  \
04467          : /*in*/    "0" (&_argvec[0])                            \
04468          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04469       );                                                          \
04470       lval = (__typeof__(lval)) _res;                             \
04471    } while (0)
04472 
04473 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
04474                                  arg7)                            \
04475    do {                                                           \
04476       volatile OrigFn        _orig = (orig);                      \
04477       volatile unsigned long _argvec[8];                          \
04478       volatile unsigned long _res;                                \
04479       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04480       _argvec[1] = (unsigned long)(arg1);                         \
04481       _argvec[2] = (unsigned long)(arg2);                         \
04482       _argvec[3] = (unsigned long)(arg3);                         \
04483       _argvec[4] = (unsigned long)(arg4);                         \
04484       _argvec[5] = (unsigned long)(arg5);                         \
04485       _argvec[6] = (unsigned long)(arg6);                         \
04486       _argvec[7] = (unsigned long)(arg7);                         \
04487       __asm__ volatile(                                           \
04488          VALGRIND_ALIGN_STACK                                     \
04489          "ldr x0, [%1, #8] \n\t"                                  \
04490          "ldr x1, [%1, #16] \n\t"                                 \
04491          "ldr x2, [%1, #24] \n\t"                                 \
04492          "ldr x3, [%1, #32] \n\t"                                 \
04493          "ldr x4, [%1, #40] \n\t"                                 \
04494          "ldr x5, [%1, #48] \n\t"                                 \
04495          "ldr x6, [%1, #56] \n\t"                                 \
04496          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04497          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04498          VALGRIND_RESTORE_STACK                                   \
04499          "mov %0, x0"                                             \
04500          : /*out*/   "=r" (_res)                                  \
04501          : /*in*/    "0" (&_argvec[0])                            \
04502          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04503       );                                                          \
04504       lval = (__typeof__(lval)) _res;                             \
04505    } while (0)
04506 
04507 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
04508                                  arg7,arg8)                       \
04509    do {                                                           \
04510       volatile OrigFn        _orig = (orig);                      \
04511       volatile unsigned long _argvec[9];                          \
04512       volatile unsigned long _res;                                \
04513       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04514       _argvec[1] = (unsigned long)(arg1);                         \
04515       _argvec[2] = (unsigned long)(arg2);                         \
04516       _argvec[3] = (unsigned long)(arg3);                         \
04517       _argvec[4] = (unsigned long)(arg4);                         \
04518       _argvec[5] = (unsigned long)(arg5);                         \
04519       _argvec[6] = (unsigned long)(arg6);                         \
04520       _argvec[7] = (unsigned long)(arg7);                         \
04521       _argvec[8] = (unsigned long)(arg8);                         \
04522       __asm__ volatile(                                           \
04523          VALGRIND_ALIGN_STACK                                     \
04524          "ldr x0, [%1, #8] \n\t"                                  \
04525          "ldr x1, [%1, #16] \n\t"                                 \
04526          "ldr x2, [%1, #24] \n\t"                                 \
04527          "ldr x3, [%1, #32] \n\t"                                 \
04528          "ldr x4, [%1, #40] \n\t"                                 \
04529          "ldr x5, [%1, #48] \n\t"                                 \
04530          "ldr x6, [%1, #56] \n\t"                                 \
04531          "ldr x7, [%1, #64] \n\t"                                 \
04532          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04533          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04534          VALGRIND_RESTORE_STACK                                   \
04535          "mov %0, x0"                                             \
04536          : /*out*/   "=r" (_res)                                  \
04537          : /*in*/    "0" (&_argvec[0])                            \
04538          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04539       );                                                          \
04540       lval = (__typeof__(lval)) _res;                             \
04541    } while (0)
04542 
04543 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
04544                                  arg7,arg8,arg9)                  \
04545    do {                                                           \
04546       volatile OrigFn        _orig = (orig);                      \
04547       volatile unsigned long _argvec[10];                         \
04548       volatile unsigned long _res;                                \
04549       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04550       _argvec[1] = (unsigned long)(arg1);                         \
04551       _argvec[2] = (unsigned long)(arg2);                         \
04552       _argvec[3] = (unsigned long)(arg3);                         \
04553       _argvec[4] = (unsigned long)(arg4);                         \
04554       _argvec[5] = (unsigned long)(arg5);                         \
04555       _argvec[6] = (unsigned long)(arg6);                         \
04556       _argvec[7] = (unsigned long)(arg7);                         \
04557       _argvec[8] = (unsigned long)(arg8);                         \
04558       _argvec[9] = (unsigned long)(arg9);                         \
04559       __asm__ volatile(                                           \
04560          VALGRIND_ALIGN_STACK                                     \
04561          "sub sp, sp, #0x20 \n\t"                                 \
04562          "ldr x0, [%1, #8] \n\t"                                  \
04563          "ldr x1, [%1, #16] \n\t"                                 \
04564          "ldr x2, [%1, #24] \n\t"                                 \
04565          "ldr x3, [%1, #32] \n\t"                                 \
04566          "ldr x4, [%1, #40] \n\t"                                 \
04567          "ldr x5, [%1, #48] \n\t"                                 \
04568          "ldr x6, [%1, #56] \n\t"                                 \
04569          "ldr x7, [%1, #64] \n\t"                                 \
04570          "ldr x8, [%1, #72] \n\t"                                 \
04571          "str x8, [sp, #0]  \n\t"                                 \
04572          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04573          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04574          VALGRIND_RESTORE_STACK                                   \
04575          "mov %0, x0"                                             \
04576          : /*out*/   "=r" (_res)                                  \
04577          : /*in*/    "0" (&_argvec[0])                            \
04578          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04579       );                                                          \
04580       lval = (__typeof__(lval)) _res;                             \
04581    } while (0)
04582 
04583 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
04584                                   arg7,arg8,arg9,arg10)           \
04585    do {                                                           \
04586       volatile OrigFn        _orig = (orig);                      \
04587       volatile unsigned long _argvec[11];                         \
04588       volatile unsigned long _res;                                \
04589       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04590       _argvec[1] = (unsigned long)(arg1);                         \
04591       _argvec[2] = (unsigned long)(arg2);                         \
04592       _argvec[3] = (unsigned long)(arg3);                         \
04593       _argvec[4] = (unsigned long)(arg4);                         \
04594       _argvec[5] = (unsigned long)(arg5);                         \
04595       _argvec[6] = (unsigned long)(arg6);                         \
04596       _argvec[7] = (unsigned long)(arg7);                         \
04597       _argvec[8] = (unsigned long)(arg8);                         \
04598       _argvec[9] = (unsigned long)(arg9);                         \
04599       _argvec[10] = (unsigned long)(arg10);                       \
04600       __asm__ volatile(                                           \
04601          VALGRIND_ALIGN_STACK                                     \
04602          "sub sp, sp, #0x20 \n\t"                                 \
04603          "ldr x0, [%1, #8] \n\t"                                  \
04604          "ldr x1, [%1, #16] \n\t"                                 \
04605          "ldr x2, [%1, #24] \n\t"                                 \
04606          "ldr x3, [%1, #32] \n\t"                                 \
04607          "ldr x4, [%1, #40] \n\t"                                 \
04608          "ldr x5, [%1, #48] \n\t"                                 \
04609          "ldr x6, [%1, #56] \n\t"                                 \
04610          "ldr x7, [%1, #64] \n\t"                                 \
04611          "ldr x8, [%1, #72] \n\t"                                 \
04612          "str x8, [sp, #0]  \n\t"                                 \
04613          "ldr x8, [%1, #80] \n\t"                                 \
04614          "str x8, [sp, #8]  \n\t"                                 \
04615          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04616          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04617          VALGRIND_RESTORE_STACK                                   \
04618          "mov %0, x0"                                             \
04619          : /*out*/   "=r" (_res)                                  \
04620          : /*in*/    "0" (&_argvec[0])                            \
04621          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04622       );                                                          \
04623       lval = (__typeof__(lval)) _res;                             \
04624    } while (0)
04625 
04626 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
04627                                   arg7,arg8,arg9,arg10,arg11)     \
04628    do {                                                           \
04629       volatile OrigFn        _orig = (orig);                      \
04630       volatile unsigned long _argvec[12];                         \
04631       volatile unsigned long _res;                                \
04632       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04633       _argvec[1] = (unsigned long)(arg1);                         \
04634       _argvec[2] = (unsigned long)(arg2);                         \
04635       _argvec[3] = (unsigned long)(arg3);                         \
04636       _argvec[4] = (unsigned long)(arg4);                         \
04637       _argvec[5] = (unsigned long)(arg5);                         \
04638       _argvec[6] = (unsigned long)(arg6);                         \
04639       _argvec[7] = (unsigned long)(arg7);                         \
04640       _argvec[8] = (unsigned long)(arg8);                         \
04641       _argvec[9] = (unsigned long)(arg9);                         \
04642       _argvec[10] = (unsigned long)(arg10);                       \
04643       _argvec[11] = (unsigned long)(arg11);                       \
04644       __asm__ volatile(                                           \
04645          VALGRIND_ALIGN_STACK                                     \
04646          "sub sp, sp, #0x30 \n\t"                                 \
04647          "ldr x0, [%1, #8] \n\t"                                  \
04648          "ldr x1, [%1, #16] \n\t"                                 \
04649          "ldr x2, [%1, #24] \n\t"                                 \
04650          "ldr x3, [%1, #32] \n\t"                                 \
04651          "ldr x4, [%1, #40] \n\t"                                 \
04652          "ldr x5, [%1, #48] \n\t"                                 \
04653          "ldr x6, [%1, #56] \n\t"                                 \
04654          "ldr x7, [%1, #64] \n\t"                                 \
04655          "ldr x8, [%1, #72] \n\t"                                 \
04656          "str x8, [sp, #0]  \n\t"                                 \
04657          "ldr x8, [%1, #80] \n\t"                                 \
04658          "str x8, [sp, #8]  \n\t"                                 \
04659          "ldr x8, [%1, #88] \n\t"                                 \
04660          "str x8, [sp, #16] \n\t"                                 \
04661          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04662          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04663          VALGRIND_RESTORE_STACK                                   \
04664          "mov %0, x0"                                             \
04665          : /*out*/   "=r" (_res)                                  \
04666          : /*in*/    "0" (&_argvec[0])                            \
04667          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04668       );                                                          \
04669       lval = (__typeof__(lval)) _res;                             \
04670    } while (0)
04671 
04672 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
04673                                   arg7,arg8,arg9,arg10,arg11,     \
04674                                   arg12)                          \
04675    do {                                                           \
04676       volatile OrigFn        _orig = (orig);                      \
04677       volatile unsigned long _argvec[13];                         \
04678       volatile unsigned long _res;                                \
04679       _argvec[0] = (unsigned long)_orig.nraddr;                   \
04680       _argvec[1] = (unsigned long)(arg1);                         \
04681       _argvec[2] = (unsigned long)(arg2);                         \
04682       _argvec[3] = (unsigned long)(arg3);                         \
04683       _argvec[4] = (unsigned long)(arg4);                         \
04684       _argvec[5] = (unsigned long)(arg5);                         \
04685       _argvec[6] = (unsigned long)(arg6);                         \
04686       _argvec[7] = (unsigned long)(arg7);                         \
04687       _argvec[8] = (unsigned long)(arg8);                         \
04688       _argvec[9] = (unsigned long)(arg9);                         \
04689       _argvec[10] = (unsigned long)(arg10);                       \
04690       _argvec[11] = (unsigned long)(arg11);                       \
04691       _argvec[12] = (unsigned long)(arg12);                       \
04692       __asm__ volatile(                                           \
04693          VALGRIND_ALIGN_STACK                                     \
04694          "sub sp, sp, #0x30 \n\t"                                 \
04695          "ldr x0, [%1, #8] \n\t"                                  \
04696          "ldr x1, [%1, #16] \n\t"                                 \
04697          "ldr x2, [%1, #24] \n\t"                                 \
04698          "ldr x3, [%1, #32] \n\t"                                 \
04699          "ldr x4, [%1, #40] \n\t"                                 \
04700          "ldr x5, [%1, #48] \n\t"                                 \
04701          "ldr x6, [%1, #56] \n\t"                                 \
04702          "ldr x7, [%1, #64] \n\t"                                 \
04703          "ldr x8, [%1, #72] \n\t"                                 \
04704          "str x8, [sp, #0]  \n\t"                                 \
04705          "ldr x8, [%1, #80] \n\t"                                 \
04706          "str x8, [sp, #8]  \n\t"                                 \
04707          "ldr x8, [%1, #88] \n\t"                                 \
04708          "str x8, [sp, #16] \n\t"                                 \
04709          "ldr x8, [%1, #96] \n\t"                                 \
04710          "str x8, [sp, #24] \n\t"                                 \
04711          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
04712          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
04713          VALGRIND_RESTORE_STACK                                   \
04714          "mov %0, x0"                                             \
04715          : /*out*/   "=r" (_res)                                  \
04716          : /*in*/    "0" (&_argvec[0])                            \
04717          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
04718       );                                                          \
04719       lval = (__typeof__(lval)) _res;                             \
04720    } while (0)
04721 
04722 #endif /* PLAT_arm64_linux */
04723 
04724 /* ------------------------- s390x-linux ------------------------- */
04725 
04726 #if defined(PLAT_s390x_linux)
04727 
04728 /* Similar workaround as amd64 (see above), but we use r11 as frame
04729    pointer and save the old r11 in r7. r11 might be used for
04730    argvec, therefore we copy argvec in r1 since r1 is clobbered
04731    after the call anyway.  */
04732 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
04733 #  define __FRAME_POINTER                                         \
04734       ,"d"(__builtin_dwarf_cfa())
04735 #  define VALGRIND_CFI_PROLOGUE                                   \
04736       ".cfi_remember_state\n\t"                                   \
04737       "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */          \
04738       "lgr 7,11\n\t"                                              \
04739       "lgr 11,%2\n\t"                                             \
04740       ".cfi_def_cfa r11, 0\n\t"
04741 #  define VALGRIND_CFI_EPILOGUE                                   \
04742       "lgr 11, 7\n\t"                                             \
04743       ".cfi_restore_state\n\t"
04744 #else
04745 #  define __FRAME_POINTER
04746 #  define VALGRIND_CFI_PROLOGUE                                   \
04747       "lgr 1,%1\n\t"
04748 #  define VALGRIND_CFI_EPILOGUE
04749 #endif
04750 
04751 /* Nb: On s390 the stack pointer is properly aligned *at all times*
04752    according to the s390 GCC maintainer. (The ABI specification is not
04753    precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and
04754    VALGRIND_RESTORE_STACK are not defined here. */
04755 
04756 /* These regs are trashed by the hidden call. Note that we overwrite
04757    r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
04758    function a proper return address. All others are ABI defined call
04759    clobbers. */
04760 #define __CALLER_SAVED_REGS "0","1","2","3","4","5","14", \
04761                            "f0","f1","f2","f3","f4","f5","f6","f7"
04762 
04763 /* Nb: Although r11 is modified in the asm snippets below (inside 
04764    VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for
04765    two reasons:
04766    (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not
04767        modified
04768    (2) GCC will complain that r11 cannot appear inside a clobber section,
04769        when compiled with -O -fno-omit-frame-pointer
04770  */
04771 
04772 #define CALL_FN_W_v(lval, orig)                                  \
04773    do {                                                          \
04774       volatile OrigFn        _orig = (orig);                     \
04775       volatile unsigned long  _argvec[1];                        \
04776       volatile unsigned long _res;                               \
04777       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04778       __asm__ volatile(                                          \
04779          VALGRIND_CFI_PROLOGUE                                   \
04780          "aghi 15,-160\n\t"                                      \
04781          "lg 1, 0(1)\n\t"  /* target->r1 */                      \
04782          VALGRIND_CALL_NOREDIR_R1                                \
04783          "lgr %0, 2\n\t"                                         \
04784          "aghi 15,160\n\t"                                       \
04785          VALGRIND_CFI_EPILOGUE                                   \
04786          : /*out*/   "=d" (_res)                                 \
04787          : /*in*/    "d" (&_argvec[0]) __FRAME_POINTER           \
04788          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
04789       );                                                         \
04790       lval = (__typeof__(lval)) _res;                            \
04791    } while (0)
04792 
04793 /* The call abi has the arguments in r2-r6 and stack */
04794 #define CALL_FN_W_W(lval, orig, arg1)                            \
04795    do {                                                          \
04796       volatile OrigFn        _orig = (orig);                     \
04797       volatile unsigned long _argvec[2];                         \
04798       volatile unsigned long _res;                               \
04799       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04800       _argvec[1] = (unsigned long)arg1;                          \
04801       __asm__ volatile(                                          \
04802          VALGRIND_CFI_PROLOGUE                                   \
04803          "aghi 15,-160\n\t"                                      \
04804          "lg 2, 8(1)\n\t"                                        \
04805          "lg 1, 0(1)\n\t"                                        \
04806          VALGRIND_CALL_NOREDIR_R1                                \
04807          "lgr %0, 2\n\t"                                         \
04808          "aghi 15,160\n\t"                                       \
04809          VALGRIND_CFI_EPILOGUE                                   \
04810          : /*out*/   "=d" (_res)                                 \
04811          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04812          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
04813       );                                                         \
04814       lval = (__typeof__(lval)) _res;                            \
04815    } while (0)
04816 
04817 #define CALL_FN_W_WW(lval, orig, arg1, arg2)                     \
04818    do {                                                          \
04819       volatile OrigFn        _orig = (orig);                     \
04820       volatile unsigned long _argvec[3];                         \
04821       volatile unsigned long _res;                               \
04822       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04823       _argvec[1] = (unsigned long)arg1;                          \
04824       _argvec[2] = (unsigned long)arg2;                          \
04825       __asm__ volatile(                                          \
04826          VALGRIND_CFI_PROLOGUE                                   \
04827          "aghi 15,-160\n\t"                                      \
04828          "lg 2, 8(1)\n\t"                                        \
04829          "lg 3,16(1)\n\t"                                        \
04830          "lg 1, 0(1)\n\t"                                        \
04831          VALGRIND_CALL_NOREDIR_R1                                \
04832          "lgr %0, 2\n\t"                                         \
04833          "aghi 15,160\n\t"                                       \
04834          VALGRIND_CFI_EPILOGUE                                   \
04835          : /*out*/   "=d" (_res)                                 \
04836          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04837          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
04838       );                                                         \
04839       lval = (__typeof__(lval)) _res;                            \
04840    } while (0)
04841 
04842 #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3)              \
04843    do {                                                          \
04844       volatile OrigFn        _orig = (orig);                     \
04845       volatile unsigned long _argvec[4];                         \
04846       volatile unsigned long _res;                               \
04847       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04848       _argvec[1] = (unsigned long)arg1;                          \
04849       _argvec[2] = (unsigned long)arg2;                          \
04850       _argvec[3] = (unsigned long)arg3;                          \
04851       __asm__ volatile(                                          \
04852          VALGRIND_CFI_PROLOGUE                                   \
04853          "aghi 15,-160\n\t"                                      \
04854          "lg 2, 8(1)\n\t"                                        \
04855          "lg 3,16(1)\n\t"                                        \
04856          "lg 4,24(1)\n\t"                                        \
04857          "lg 1, 0(1)\n\t"                                        \
04858          VALGRIND_CALL_NOREDIR_R1                                \
04859          "lgr %0, 2\n\t"                                         \
04860          "aghi 15,160\n\t"                                       \
04861          VALGRIND_CFI_EPILOGUE                                   \
04862          : /*out*/   "=d" (_res)                                 \
04863          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04864          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
04865       );                                                         \
04866       lval = (__typeof__(lval)) _res;                            \
04867    } while (0)
04868 
04869 #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4)       \
04870    do {                                                          \
04871       volatile OrigFn        _orig = (orig);                     \
04872       volatile unsigned long _argvec[5];                         \
04873       volatile unsigned long _res;                               \
04874       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04875       _argvec[1] = (unsigned long)arg1;                          \
04876       _argvec[2] = (unsigned long)arg2;                          \
04877       _argvec[3] = (unsigned long)arg3;                          \
04878       _argvec[4] = (unsigned long)arg4;                          \
04879       __asm__ volatile(                                          \
04880          VALGRIND_CFI_PROLOGUE                                   \
04881          "aghi 15,-160\n\t"                                      \
04882          "lg 2, 8(1)\n\t"                                        \
04883          "lg 3,16(1)\n\t"                                        \
04884          "lg 4,24(1)\n\t"                                        \
04885          "lg 5,32(1)\n\t"                                        \
04886          "lg 1, 0(1)\n\t"                                        \
04887          VALGRIND_CALL_NOREDIR_R1                                \
04888          "lgr %0, 2\n\t"                                         \
04889          "aghi 15,160\n\t"                                       \
04890          VALGRIND_CFI_EPILOGUE                                   \
04891          : /*out*/   "=d" (_res)                                 \
04892          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04893          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
04894       );                                                         \
04895       lval = (__typeof__(lval)) _res;                            \
04896    } while (0)
04897 
04898 #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5)   \
04899    do {                                                          \
04900       volatile OrigFn        _orig = (orig);                     \
04901       volatile unsigned long _argvec[6];                         \
04902       volatile unsigned long _res;                               \
04903       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04904       _argvec[1] = (unsigned long)arg1;                          \
04905       _argvec[2] = (unsigned long)arg2;                          \
04906       _argvec[3] = (unsigned long)arg3;                          \
04907       _argvec[4] = (unsigned long)arg4;                          \
04908       _argvec[5] = (unsigned long)arg5;                          \
04909       __asm__ volatile(                                          \
04910          VALGRIND_CFI_PROLOGUE                                   \
04911          "aghi 15,-160\n\t"                                      \
04912          "lg 2, 8(1)\n\t"                                        \
04913          "lg 3,16(1)\n\t"                                        \
04914          "lg 4,24(1)\n\t"                                        \
04915          "lg 5,32(1)\n\t"                                        \
04916          "lg 6,40(1)\n\t"                                        \
04917          "lg 1, 0(1)\n\t"                                        \
04918          VALGRIND_CALL_NOREDIR_R1                                \
04919          "lgr %0, 2\n\t"                                         \
04920          "aghi 15,160\n\t"                                       \
04921          VALGRIND_CFI_EPILOGUE                                   \
04922          : /*out*/   "=d" (_res)                                 \
04923          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04924          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
04925       );                                                         \
04926       lval = (__typeof__(lval)) _res;                            \
04927    } while (0)
04928 
04929 #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
04930                      arg6)                                       \
04931    do {                                                          \
04932       volatile OrigFn        _orig = (orig);                     \
04933       volatile unsigned long _argvec[7];                         \
04934       volatile unsigned long _res;                               \
04935       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04936       _argvec[1] = (unsigned long)arg1;                          \
04937       _argvec[2] = (unsigned long)arg2;                          \
04938       _argvec[3] = (unsigned long)arg3;                          \
04939       _argvec[4] = (unsigned long)arg4;                          \
04940       _argvec[5] = (unsigned long)arg5;                          \
04941       _argvec[6] = (unsigned long)arg6;                          \
04942       __asm__ volatile(                                          \
04943          VALGRIND_CFI_PROLOGUE                                   \
04944          "aghi 15,-168\n\t"                                      \
04945          "lg 2, 8(1)\n\t"                                        \
04946          "lg 3,16(1)\n\t"                                        \
04947          "lg 4,24(1)\n\t"                                        \
04948          "lg 5,32(1)\n\t"                                        \
04949          "lg 6,40(1)\n\t"                                        \
04950          "mvc 160(8,15), 48(1)\n\t"                              \
04951          "lg 1, 0(1)\n\t"                                        \
04952          VALGRIND_CALL_NOREDIR_R1                                \
04953          "lgr %0, 2\n\t"                                         \
04954          "aghi 15,168\n\t"                                       \
04955          VALGRIND_CFI_EPILOGUE                                   \
04956          : /*out*/   "=d" (_res)                                 \
04957          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04958          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
04959       );                                                         \
04960       lval = (__typeof__(lval)) _res;                            \
04961    } while (0)
04962 
04963 #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
04964                      arg6, arg7)                                 \
04965    do {                                                          \
04966       volatile OrigFn        _orig = (orig);                     \
04967       volatile unsigned long _argvec[8];                         \
04968       volatile unsigned long _res;                               \
04969       _argvec[0] = (unsigned long)_orig.nraddr;                  \
04970       _argvec[1] = (unsigned long)arg1;                          \
04971       _argvec[2] = (unsigned long)arg2;                          \
04972       _argvec[3] = (unsigned long)arg3;                          \
04973       _argvec[4] = (unsigned long)arg4;                          \
04974       _argvec[5] = (unsigned long)arg5;                          \
04975       _argvec[6] = (unsigned long)arg6;                          \
04976       _argvec[7] = (unsigned long)arg7;                          \
04977       __asm__ volatile(                                          \
04978          VALGRIND_CFI_PROLOGUE                                   \
04979          "aghi 15,-176\n\t"                                      \
04980          "lg 2, 8(1)\n\t"                                        \
04981          "lg 3,16(1)\n\t"                                        \
04982          "lg 4,24(1)\n\t"                                        \
04983          "lg 5,32(1)\n\t"                                        \
04984          "lg 6,40(1)\n\t"                                        \
04985          "mvc 160(8,15), 48(1)\n\t"                              \
04986          "mvc 168(8,15), 56(1)\n\t"                              \
04987          "lg 1, 0(1)\n\t"                                        \
04988          VALGRIND_CALL_NOREDIR_R1                                \
04989          "lgr %0, 2\n\t"                                         \
04990          "aghi 15,176\n\t"                                       \
04991          VALGRIND_CFI_EPILOGUE                                   \
04992          : /*out*/   "=d" (_res)                                 \
04993          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
04994          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
04995       );                                                         \
04996       lval = (__typeof__(lval)) _res;                            \
04997    } while (0)
04998 
04999 #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
05000                      arg6, arg7 ,arg8)                           \
05001    do {                                                          \
05002       volatile OrigFn        _orig = (orig);                     \
05003       volatile unsigned long _argvec[9];                         \
05004       volatile unsigned long _res;                               \
05005       _argvec[0] = (unsigned long)_orig.nraddr;                  \
05006       _argvec[1] = (unsigned long)arg1;                          \
05007       _argvec[2] = (unsigned long)arg2;                          \
05008       _argvec[3] = (unsigned long)arg3;                          \
05009       _argvec[4] = (unsigned long)arg4;                          \
05010       _argvec[5] = (unsigned long)arg5;                          \
05011       _argvec[6] = (unsigned long)arg6;                          \
05012       _argvec[7] = (unsigned long)arg7;                          \
05013       _argvec[8] = (unsigned long)arg8;                          \
05014       __asm__ volatile(                                          \
05015          VALGRIND_CFI_PROLOGUE                                   \
05016          "aghi 15,-184\n\t"                                      \
05017          "lg 2, 8(1)\n\t"                                        \
05018          "lg 3,16(1)\n\t"                                        \
05019          "lg 4,24(1)\n\t"                                        \
05020          "lg 5,32(1)\n\t"                                        \
05021          "lg 6,40(1)\n\t"                                        \
05022          "mvc 160(8,15), 48(1)\n\t"                              \
05023          "mvc 168(8,15), 56(1)\n\t"                              \
05024          "mvc 176(8,15), 64(1)\n\t"                              \
05025          "lg 1, 0(1)\n\t"                                        \
05026          VALGRIND_CALL_NOREDIR_R1                                \
05027          "lgr %0, 2\n\t"                                         \
05028          "aghi 15,184\n\t"                                       \
05029          VALGRIND_CFI_EPILOGUE                                   \
05030          : /*out*/   "=d" (_res)                                 \
05031          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
05032          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
05033       );                                                         \
05034       lval = (__typeof__(lval)) _res;                            \
05035    } while (0)
05036 
05037 #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
05038                      arg6, arg7 ,arg8, arg9)                     \
05039    do {                                                          \
05040       volatile OrigFn        _orig = (orig);                     \
05041       volatile unsigned long _argvec[10];                        \
05042       volatile unsigned long _res;                               \
05043       _argvec[0] = (unsigned long)_orig.nraddr;                  \
05044       _argvec[1] = (unsigned long)arg1;                          \
05045       _argvec[2] = (unsigned long)arg2;                          \
05046       _argvec[3] = (unsigned long)arg3;                          \
05047       _argvec[4] = (unsigned long)arg4;                          \
05048       _argvec[5] = (unsigned long)arg5;                          \
05049       _argvec[6] = (unsigned long)arg6;                          \
05050       _argvec[7] = (unsigned long)arg7;                          \
05051       _argvec[8] = (unsigned long)arg8;                          \
05052       _argvec[9] = (unsigned long)arg9;                          \
05053       __asm__ volatile(                                          \
05054          VALGRIND_CFI_PROLOGUE                                   \
05055          "aghi 15,-192\n\t"                                      \
05056          "lg 2, 8(1)\n\t"                                        \
05057          "lg 3,16(1)\n\t"                                        \
05058          "lg 4,24(1)\n\t"                                        \
05059          "lg 5,32(1)\n\t"                                        \
05060          "lg 6,40(1)\n\t"                                        \
05061          "mvc 160(8,15), 48(1)\n\t"                              \
05062          "mvc 168(8,15), 56(1)\n\t"                              \
05063          "mvc 176(8,15), 64(1)\n\t"                              \
05064          "mvc 184(8,15), 72(1)\n\t"                              \
05065          "lg 1, 0(1)\n\t"                                        \
05066          VALGRIND_CALL_NOREDIR_R1                                \
05067          "lgr %0, 2\n\t"                                         \
05068          "aghi 15,192\n\t"                                       \
05069          VALGRIND_CFI_EPILOGUE                                   \
05070          : /*out*/   "=d" (_res)                                 \
05071          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
05072          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
05073       );                                                         \
05074       lval = (__typeof__(lval)) _res;                            \
05075    } while (0)
05076 
05077 #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
05078                      arg6, arg7 ,arg8, arg9, arg10)              \
05079    do {                                                          \
05080       volatile OrigFn        _orig = (orig);                     \
05081       volatile unsigned long _argvec[11];                        \
05082       volatile unsigned long _res;                               \
05083       _argvec[0] = (unsigned long)_orig.nraddr;                  \
05084       _argvec[1] = (unsigned long)arg1;                          \
05085       _argvec[2] = (unsigned long)arg2;                          \
05086       _argvec[3] = (unsigned long)arg3;                          \
05087       _argvec[4] = (unsigned long)arg4;                          \
05088       _argvec[5] = (unsigned long)arg5;                          \
05089       _argvec[6] = (unsigned long)arg6;                          \
05090       _argvec[7] = (unsigned long)arg7;                          \
05091       _argvec[8] = (unsigned long)arg8;                          \
05092       _argvec[9] = (unsigned long)arg9;                          \
05093       _argvec[10] = (unsigned long)arg10;                        \
05094       __asm__ volatile(                                          \
05095          VALGRIND_CFI_PROLOGUE                                   \
05096          "aghi 15,-200\n\t"                                      \
05097          "lg 2, 8(1)\n\t"                                        \
05098          "lg 3,16(1)\n\t"                                        \
05099          "lg 4,24(1)\n\t"                                        \
05100          "lg 5,32(1)\n\t"                                        \
05101          "lg 6,40(1)\n\t"                                        \
05102          "mvc 160(8,15), 48(1)\n\t"                              \
05103          "mvc 168(8,15), 56(1)\n\t"                              \
05104          "mvc 176(8,15), 64(1)\n\t"                              \
05105          "mvc 184(8,15), 72(1)\n\t"                              \
05106          "mvc 192(8,15), 80(1)\n\t"                              \
05107          "lg 1, 0(1)\n\t"                                        \
05108          VALGRIND_CALL_NOREDIR_R1                                \
05109          "lgr %0, 2\n\t"                                         \
05110          "aghi 15,200\n\t"                                       \
05111          VALGRIND_CFI_EPILOGUE                                   \
05112          : /*out*/   "=d" (_res)                                 \
05113          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
05114          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
05115       );                                                         \
05116       lval = (__typeof__(lval)) _res;                            \
05117    } while (0)
05118 
05119 #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
05120                      arg6, arg7 ,arg8, arg9, arg10, arg11)       \
05121    do {                                                          \
05122       volatile OrigFn        _orig = (orig);                     \
05123       volatile unsigned long _argvec[12];                        \
05124       volatile unsigned long _res;                               \
05125       _argvec[0] = (unsigned long)_orig.nraddr;                  \
05126       _argvec[1] = (unsigned long)arg1;                          \
05127       _argvec[2] = (unsigned long)arg2;                          \
05128       _argvec[3] = (unsigned long)arg3;                          \
05129       _argvec[4] = (unsigned long)arg4;                          \
05130       _argvec[5] = (unsigned long)arg5;                          \
05131       _argvec[6] = (unsigned long)arg6;                          \
05132       _argvec[7] = (unsigned long)arg7;                          \
05133       _argvec[8] = (unsigned long)arg8;                          \
05134       _argvec[9] = (unsigned long)arg9;                          \
05135       _argvec[10] = (unsigned long)arg10;                        \
05136       _argvec[11] = (unsigned long)arg11;                        \
05137       __asm__ volatile(                                          \
05138          VALGRIND_CFI_PROLOGUE                                   \
05139          "aghi 15,-208\n\t"                                      \
05140          "lg 2, 8(1)\n\t"                                        \
05141          "lg 3,16(1)\n\t"                                        \
05142          "lg 4,24(1)\n\t"                                        \
05143          "lg 5,32(1)\n\t"                                        \
05144          "lg 6,40(1)\n\t"                                        \
05145          "mvc 160(8,15), 48(1)\n\t"                              \
05146          "mvc 168(8,15), 56(1)\n\t"                              \
05147          "mvc 176(8,15), 64(1)\n\t"                              \
05148          "mvc 184(8,15), 72(1)\n\t"                              \
05149          "mvc 192(8,15), 80(1)\n\t"                              \
05150          "mvc 200(8,15), 88(1)\n\t"                              \
05151          "lg 1, 0(1)\n\t"                                        \
05152          VALGRIND_CALL_NOREDIR_R1                                \
05153          "lgr %0, 2\n\t"                                         \
05154          "aghi 15,208\n\t"                                       \
05155          VALGRIND_CFI_EPILOGUE                                   \
05156          : /*out*/   "=d" (_res)                                 \
05157          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
05158          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
05159       );                                                         \
05160       lval = (__typeof__(lval)) _res;                            \
05161    } while (0)
05162 
05163 #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
05164                      arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
05165    do {                                                          \
05166       volatile OrigFn        _orig = (orig);                     \
05167       volatile unsigned long _argvec[13];                        \
05168       volatile unsigned long _res;                               \
05169       _argvec[0] = (unsigned long)_orig.nraddr;                  \
05170       _argvec[1] = (unsigned long)arg1;                          \
05171       _argvec[2] = (unsigned long)arg2;                          \
05172       _argvec[3] = (unsigned long)arg3;                          \
05173       _argvec[4] = (unsigned long)arg4;                          \
05174       _argvec[5] = (unsigned long)arg5;                          \
05175       _argvec[6] = (unsigned long)arg6;                          \
05176       _argvec[7] = (unsigned long)arg7;                          \
05177       _argvec[8] = (unsigned long)arg8;                          \
05178       _argvec[9] = (unsigned long)arg9;                          \
05179       _argvec[10] = (unsigned long)arg10;                        \
05180       _argvec[11] = (unsigned long)arg11;                        \
05181       _argvec[12] = (unsigned long)arg12;                        \
05182       __asm__ volatile(                                          \
05183          VALGRIND_CFI_PROLOGUE                                   \
05184          "aghi 15,-216\n\t"                                      \
05185          "lg 2, 8(1)\n\t"                                        \
05186          "lg 3,16(1)\n\t"                                        \
05187          "lg 4,24(1)\n\t"                                        \
05188          "lg 5,32(1)\n\t"                                        \
05189          "lg 6,40(1)\n\t"                                        \
05190          "mvc 160(8,15), 48(1)\n\t"                              \
05191          "mvc 168(8,15), 56(1)\n\t"                              \
05192          "mvc 176(8,15), 64(1)\n\t"                              \
05193          "mvc 184(8,15), 72(1)\n\t"                              \
05194          "mvc 192(8,15), 80(1)\n\t"                              \
05195          "mvc 200(8,15), 88(1)\n\t"                              \
05196          "mvc 208(8,15), 96(1)\n\t"                              \
05197          "lg 1, 0(1)\n\t"                                        \
05198          VALGRIND_CALL_NOREDIR_R1                                \
05199          "lgr %0, 2\n\t"                                         \
05200          "aghi 15,216\n\t"                                       \
05201          VALGRIND_CFI_EPILOGUE                                   \
05202          : /*out*/   "=d" (_res)                                 \
05203          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
05204          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
05205       );                                                         \
05206       lval = (__typeof__(lval)) _res;                            \
05207    } while (0)
05208 
05209 
05210 #endif /* PLAT_s390x_linux */
05211 
05212 /* ------------------------- mips32-linux ----------------------- */
05213  
05214 #if defined(PLAT_mips32_linux)
05215 
05216 /* These regs are trashed by the hidden call. */
05217 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
05218 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
05219 "$25", "$31"
05220 
05221 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
05222    long) == 4. */
05223 
05224 #define CALL_FN_W_v(lval, orig)                                   \
05225    do {                                                           \
05226       volatile OrigFn        _orig = (orig);                      \
05227       volatile unsigned long _argvec[1];                          \
05228       volatile unsigned long _res;                                \
05229       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05230       __asm__ volatile(                                           \
05231          "subu $29, $29, 8 \n\t"                                  \
05232          "sw $28, 0($29) \n\t"                                    \
05233          "sw $31, 4($29) \n\t"                                    \
05234          "subu $29, $29, 16 \n\t"                                 \
05235          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05236          VALGRIND_CALL_NOREDIR_T9                                 \
05237          "addu $29, $29, 16\n\t"                                  \
05238          "lw $28, 0($29) \n\t"                                    \
05239          "lw $31, 4($29) \n\t"                                    \
05240          "addu $29, $29, 8 \n\t"                                  \
05241          "move %0, $2\n"                                          \
05242          : /*out*/   "=r" (_res)                                  \
05243          : /*in*/    "0" (&_argvec[0])                            \
05244          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05245       );                                                          \
05246       lval = (__typeof__(lval)) _res;                             \
05247    } while (0)
05248 
05249 #define CALL_FN_W_W(lval, orig, arg1)                             \
05250    do {                                                           \
05251       volatile OrigFn        _orig = (orig);                      \
05252      volatile unsigned long _argvec[2];                           \
05253       volatile unsigned long _res;                                \
05254       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05255       _argvec[1] = (unsigned long)(arg1);                         \
05256       __asm__ volatile(                                           \
05257          "subu $29, $29, 8 \n\t"                                  \
05258          "sw $28, 0($29) \n\t"                                    \
05259          "sw $31, 4($29) \n\t"                                    \
05260          "subu $29, $29, 16 \n\t"                                 \
05261          "lw $4, 4(%1) \n\t"   /* arg1*/                          \
05262          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05263          VALGRIND_CALL_NOREDIR_T9                                 \
05264          "addu $29, $29, 16 \n\t"                                 \
05265          "lw $28, 0($29) \n\t"                                    \
05266          "lw $31, 4($29) \n\t"                                    \
05267          "addu $29, $29, 8 \n\t"                                  \
05268          "move %0, $2\n"                                          \
05269          : /*out*/   "=r" (_res)                                  \
05270          : /*in*/    "0" (&_argvec[0])                            \
05271          : /*trash*/ "memory",  __CALLER_SAVED_REGS               \
05272       );                                                          \
05273       lval = (__typeof__(lval)) _res;                             \
05274    } while (0)
05275 
05276 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
05277    do {                                                           \
05278       volatile OrigFn        _orig = (orig);                      \
05279       volatile unsigned long _argvec[3];                          \
05280       volatile unsigned long _res;                                \
05281       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05282       _argvec[1] = (unsigned long)(arg1);                         \
05283       _argvec[2] = (unsigned long)(arg2);                         \
05284       __asm__ volatile(                                           \
05285          "subu $29, $29, 8 \n\t"                                  \
05286          "sw $28, 0($29) \n\t"                                    \
05287          "sw $31, 4($29) \n\t"                                    \
05288          "subu $29, $29, 16 \n\t"                                 \
05289          "lw $4, 4(%1) \n\t"                                      \
05290          "lw $5, 8(%1) \n\t"                                      \
05291          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05292          VALGRIND_CALL_NOREDIR_T9                                 \
05293          "addu $29, $29, 16 \n\t"                                 \
05294          "lw $28, 0($29) \n\t"                                    \
05295          "lw $31, 4($29) \n\t"                                    \
05296          "addu $29, $29, 8 \n\t"                                  \
05297          "move %0, $2\n"                                          \
05298          : /*out*/   "=r" (_res)                                  \
05299          : /*in*/    "0" (&_argvec[0])                            \
05300          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05301       );                                                          \
05302       lval = (__typeof__(lval)) _res;                             \
05303    } while (0)
05304 
05305 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
05306    do {                                                           \
05307       volatile OrigFn        _orig = (orig);                      \
05308       volatile unsigned long _argvec[4];                          \
05309       volatile unsigned long _res;                                \
05310       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05311       _argvec[1] = (unsigned long)(arg1);                         \
05312       _argvec[2] = (unsigned long)(arg2);                         \
05313       _argvec[3] = (unsigned long)(arg3);                         \
05314       __asm__ volatile(                                           \
05315          "subu $29, $29, 8 \n\t"                                  \
05316          "sw $28, 0($29) \n\t"                                    \
05317          "sw $31, 4($29) \n\t"                                    \
05318          "subu $29, $29, 16 \n\t"                                 \
05319          "lw $4, 4(%1) \n\t"                                      \
05320          "lw $5, 8(%1) \n\t"                                      \
05321          "lw $6, 12(%1) \n\t"                                     \
05322          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05323          VALGRIND_CALL_NOREDIR_T9                                 \
05324          "addu $29, $29, 16 \n\t"                                 \
05325          "lw $28, 0($29) \n\t"                                    \
05326          "lw $31, 4($29) \n\t"                                    \
05327          "addu $29, $29, 8 \n\t"                                  \
05328          "move %0, $2\n"                                          \
05329          : /*out*/   "=r" (_res)                                  \
05330          : /*in*/    "0" (&_argvec[0])                            \
05331          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05332       );                                                          \
05333       lval = (__typeof__(lval)) _res;                             \
05334    } while (0)
05335 
05336 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
05337    do {                                                           \
05338       volatile OrigFn        _orig = (orig);                      \
05339       volatile unsigned long _argvec[5];                          \
05340       volatile unsigned long _res;                                \
05341       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05342       _argvec[1] = (unsigned long)(arg1);                         \
05343       _argvec[2] = (unsigned long)(arg2);                         \
05344       _argvec[3] = (unsigned long)(arg3);                         \
05345       _argvec[4] = (unsigned long)(arg4);                         \
05346       __asm__ volatile(                                           \
05347          "subu $29, $29, 8 \n\t"                                  \
05348          "sw $28, 0($29) \n\t"                                    \
05349          "sw $31, 4($29) \n\t"                                    \
05350          "subu $29, $29, 16 \n\t"                                 \
05351          "lw $4, 4(%1) \n\t"                                      \
05352          "lw $5, 8(%1) \n\t"                                      \
05353          "lw $6, 12(%1) \n\t"                                     \
05354          "lw $7, 16(%1) \n\t"                                     \
05355          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05356          VALGRIND_CALL_NOREDIR_T9                                 \
05357          "addu $29, $29, 16 \n\t"                                 \
05358          "lw $28, 0($29) \n\t"                                    \
05359          "lw $31, 4($29) \n\t"                                    \
05360          "addu $29, $29, 8 \n\t"                                  \
05361          "move %0, $2\n"                                          \
05362          : /*out*/   "=r" (_res)                                  \
05363          : /*in*/    "0" (&_argvec[0])                            \
05364          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05365       );                                                          \
05366       lval = (__typeof__(lval)) _res;                             \
05367    } while (0)
05368 
05369 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
05370    do {                                                           \
05371       volatile OrigFn        _orig = (orig);                      \
05372       volatile unsigned long _argvec[6];                          \
05373       volatile unsigned long _res;                                \
05374       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05375       _argvec[1] = (unsigned long)(arg1);                         \
05376       _argvec[2] = (unsigned long)(arg2);                         \
05377       _argvec[3] = (unsigned long)(arg3);                         \
05378       _argvec[4] = (unsigned long)(arg4);                         \
05379       _argvec[5] = (unsigned long)(arg5);                         \
05380       __asm__ volatile(                                           \
05381          "subu $29, $29, 8 \n\t"                                  \
05382          "sw $28, 0($29) \n\t"                                    \
05383          "sw $31, 4($29) \n\t"                                    \
05384          "lw $4, 20(%1) \n\t"                                     \
05385          "subu $29, $29, 24\n\t"                                  \
05386          "sw $4, 16($29) \n\t"                                    \
05387          "lw $4, 4(%1) \n\t"                                      \
05388          "lw $5, 8(%1) \n\t"                                      \
05389          "lw $6, 12(%1) \n\t"                                     \
05390          "lw $7, 16(%1) \n\t"                                     \
05391          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05392          VALGRIND_CALL_NOREDIR_T9                                 \
05393          "addu $29, $29, 24 \n\t"                                 \
05394          "lw $28, 0($29) \n\t"                                    \
05395          "lw $31, 4($29) \n\t"                                    \
05396          "addu $29, $29, 8 \n\t"                                  \
05397          "move %0, $2\n"                                          \
05398          : /*out*/   "=r" (_res)                                  \
05399          : /*in*/    "0" (&_argvec[0])                            \
05400          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05401       );                                                          \
05402       lval = (__typeof__(lval)) _res;                             \
05403    } while (0)
05404 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
05405    do {                                                           \
05406       volatile OrigFn        _orig = (orig);                      \
05407       volatile unsigned long _argvec[7];                          \
05408       volatile unsigned long _res;                                \
05409       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05410       _argvec[1] = (unsigned long)(arg1);                         \
05411       _argvec[2] = (unsigned long)(arg2);                         \
05412       _argvec[3] = (unsigned long)(arg3);                         \
05413       _argvec[4] = (unsigned long)(arg4);                         \
05414       _argvec[5] = (unsigned long)(arg5);                         \
05415       _argvec[6] = (unsigned long)(arg6);                         \
05416       __asm__ volatile(                                           \
05417          "subu $29, $29, 8 \n\t"                                  \
05418          "sw $28, 0($29) \n\t"                                    \
05419          "sw $31, 4($29) \n\t"                                    \
05420          "lw $4, 20(%1) \n\t"                                     \
05421          "subu $29, $29, 32\n\t"                                  \
05422          "sw $4, 16($29) \n\t"                                    \
05423          "lw $4, 24(%1) \n\t"                                     \
05424          "nop\n\t"                                                \
05425          "sw $4, 20($29) \n\t"                                    \
05426          "lw $4, 4(%1) \n\t"                                      \
05427          "lw $5, 8(%1) \n\t"                                      \
05428          "lw $6, 12(%1) \n\t"                                     \
05429          "lw $7, 16(%1) \n\t"                                     \
05430          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05431          VALGRIND_CALL_NOREDIR_T9                                 \
05432          "addu $29, $29, 32 \n\t"                                 \
05433          "lw $28, 0($29) \n\t"                                    \
05434          "lw $31, 4($29) \n\t"                                    \
05435          "addu $29, $29, 8 \n\t"                                  \
05436          "move %0, $2\n"                                          \
05437          : /*out*/   "=r" (_res)                                  \
05438          : /*in*/    "0" (&_argvec[0])                            \
05439          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05440       );                                                          \
05441       lval = (__typeof__(lval)) _res;                             \
05442    } while (0)
05443 
05444 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
05445                                  arg7)                            \
05446    do {                                                           \
05447       volatile OrigFn        _orig = (orig);                      \
05448       volatile unsigned long _argvec[8];                          \
05449       volatile unsigned long _res;                                \
05450       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05451       _argvec[1] = (unsigned long)(arg1);                         \
05452       _argvec[2] = (unsigned long)(arg2);                         \
05453       _argvec[3] = (unsigned long)(arg3);                         \
05454       _argvec[4] = (unsigned long)(arg4);                         \
05455       _argvec[5] = (unsigned long)(arg5);                         \
05456       _argvec[6] = (unsigned long)(arg6);                         \
05457       _argvec[7] = (unsigned long)(arg7);                         \
05458       __asm__ volatile(                                           \
05459          "subu $29, $29, 8 \n\t"                                  \
05460          "sw $28, 0($29) \n\t"                                    \
05461          "sw $31, 4($29) \n\t"                                    \
05462          "lw $4, 20(%1) \n\t"                                     \
05463          "subu $29, $29, 32\n\t"                                  \
05464          "sw $4, 16($29) \n\t"                                    \
05465          "lw $4, 24(%1) \n\t"                                     \
05466          "sw $4, 20($29) \n\t"                                    \
05467          "lw $4, 28(%1) \n\t"                                     \
05468          "sw $4, 24($29) \n\t"                                    \
05469          "lw $4, 4(%1) \n\t"                                      \
05470          "lw $5, 8(%1) \n\t"                                      \
05471          "lw $6, 12(%1) \n\t"                                     \
05472          "lw $7, 16(%1) \n\t"                                     \
05473          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05474          VALGRIND_CALL_NOREDIR_T9                                 \
05475          "addu $29, $29, 32 \n\t"                                 \
05476          "lw $28, 0($29) \n\t"                                    \
05477          "lw $31, 4($29) \n\t"                                    \
05478          "addu $29, $29, 8 \n\t"                                  \
05479          "move %0, $2\n"                                          \
05480          : /*out*/   "=r" (_res)                                  \
05481          : /*in*/    "0" (&_argvec[0])                            \
05482          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05483       );                                                          \
05484       lval = (__typeof__(lval)) _res;                             \
05485    } while (0)
05486 
05487 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
05488                                  arg7,arg8)                       \
05489    do {                                                           \
05490       volatile OrigFn        _orig = (orig);                      \
05491       volatile unsigned long _argvec[9];                          \
05492       volatile unsigned long _res;                                \
05493       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05494       _argvec[1] = (unsigned long)(arg1);                         \
05495       _argvec[2] = (unsigned long)(arg2);                         \
05496       _argvec[3] = (unsigned long)(arg3);                         \
05497       _argvec[4] = (unsigned long)(arg4);                         \
05498       _argvec[5] = (unsigned long)(arg5);                         \
05499       _argvec[6] = (unsigned long)(arg6);                         \
05500       _argvec[7] = (unsigned long)(arg7);                         \
05501       _argvec[8] = (unsigned long)(arg8);                         \
05502       __asm__ volatile(                                           \
05503          "subu $29, $29, 8 \n\t"                                  \
05504          "sw $28, 0($29) \n\t"                                    \
05505          "sw $31, 4($29) \n\t"                                    \
05506          "lw $4, 20(%1) \n\t"                                     \
05507          "subu $29, $29, 40\n\t"                                  \
05508          "sw $4, 16($29) \n\t"                                    \
05509          "lw $4, 24(%1) \n\t"                                     \
05510          "sw $4, 20($29) \n\t"                                    \
05511          "lw $4, 28(%1) \n\t"                                     \
05512          "sw $4, 24($29) \n\t"                                    \
05513          "lw $4, 32(%1) \n\t"                                     \
05514          "sw $4, 28($29) \n\t"                                    \
05515          "lw $4, 4(%1) \n\t"                                      \
05516          "lw $5, 8(%1) \n\t"                                      \
05517          "lw $6, 12(%1) \n\t"                                     \
05518          "lw $7, 16(%1) \n\t"                                     \
05519          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05520          VALGRIND_CALL_NOREDIR_T9                                 \
05521          "addu $29, $29, 40 \n\t"                                 \
05522          "lw $28, 0($29) \n\t"                                    \
05523          "lw $31, 4($29) \n\t"                                    \
05524          "addu $29, $29, 8 \n\t"                                  \
05525          "move %0, $2\n"                                          \
05526          : /*out*/   "=r" (_res)                                  \
05527          : /*in*/    "0" (&_argvec[0])                            \
05528          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05529       );                                                          \
05530       lval = (__typeof__(lval)) _res;                             \
05531    } while (0)
05532 
05533 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
05534                                  arg7,arg8,arg9)                  \
05535    do {                                                           \
05536       volatile OrigFn        _orig = (orig);                      \
05537       volatile unsigned long _argvec[10];                         \
05538       volatile unsigned long _res;                                \
05539       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05540       _argvec[1] = (unsigned long)(arg1);                         \
05541       _argvec[2] = (unsigned long)(arg2);                         \
05542       _argvec[3] = (unsigned long)(arg3);                         \
05543       _argvec[4] = (unsigned long)(arg4);                         \
05544       _argvec[5] = (unsigned long)(arg5);                         \
05545       _argvec[6] = (unsigned long)(arg6);                         \
05546       _argvec[7] = (unsigned long)(arg7);                         \
05547       _argvec[8] = (unsigned long)(arg8);                         \
05548       _argvec[9] = (unsigned long)(arg9);                         \
05549       __asm__ volatile(                                           \
05550          "subu $29, $29, 8 \n\t"                                  \
05551          "sw $28, 0($29) \n\t"                                    \
05552          "sw $31, 4($29) \n\t"                                    \
05553          "lw $4, 20(%1) \n\t"                                     \
05554          "subu $29, $29, 40\n\t"                                  \
05555          "sw $4, 16($29) \n\t"                                    \
05556          "lw $4, 24(%1) \n\t"                                     \
05557          "sw $4, 20($29) \n\t"                                    \
05558          "lw $4, 28(%1) \n\t"                                     \
05559          "sw $4, 24($29) \n\t"                                    \
05560          "lw $4, 32(%1) \n\t"                                     \
05561          "sw $4, 28($29) \n\t"                                    \
05562          "lw $4, 36(%1) \n\t"                                     \
05563          "sw $4, 32($29) \n\t"                                    \
05564          "lw $4, 4(%1) \n\t"                                      \
05565          "lw $5, 8(%1) \n\t"                                      \
05566          "lw $6, 12(%1) \n\t"                                     \
05567          "lw $7, 16(%1) \n\t"                                     \
05568          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05569          VALGRIND_CALL_NOREDIR_T9                                 \
05570          "addu $29, $29, 40 \n\t"                                 \
05571          "lw $28, 0($29) \n\t"                                    \
05572          "lw $31, 4($29) \n\t"                                    \
05573          "addu $29, $29, 8 \n\t"                                  \
05574          "move %0, $2\n"                                          \
05575          : /*out*/   "=r" (_res)                                  \
05576          : /*in*/    "0" (&_argvec[0])                            \
05577          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05578       );                                                          \
05579       lval = (__typeof__(lval)) _res;                             \
05580    } while (0)
05581 
05582 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
05583                                   arg7,arg8,arg9,arg10)           \
05584    do {                                                           \
05585       volatile OrigFn        _orig = (orig);                      \
05586       volatile unsigned long _argvec[11];                         \
05587       volatile unsigned long _res;                                \
05588       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05589       _argvec[1] = (unsigned long)(arg1);                         \
05590       _argvec[2] = (unsigned long)(arg2);                         \
05591       _argvec[3] = (unsigned long)(arg3);                         \
05592       _argvec[4] = (unsigned long)(arg4);                         \
05593       _argvec[5] = (unsigned long)(arg5);                         \
05594       _argvec[6] = (unsigned long)(arg6);                         \
05595       _argvec[7] = (unsigned long)(arg7);                         \
05596       _argvec[8] = (unsigned long)(arg8);                         \
05597       _argvec[9] = (unsigned long)(arg9);                         \
05598       _argvec[10] = (unsigned long)(arg10);                       \
05599       __asm__ volatile(                                           \
05600          "subu $29, $29, 8 \n\t"                                  \
05601          "sw $28, 0($29) \n\t"                                    \
05602          "sw $31, 4($29) \n\t"                                    \
05603          "lw $4, 20(%1) \n\t"                                     \
05604          "subu $29, $29, 48\n\t"                                  \
05605          "sw $4, 16($29) \n\t"                                    \
05606          "lw $4, 24(%1) \n\t"                                     \
05607          "sw $4, 20($29) \n\t"                                    \
05608          "lw $4, 28(%1) \n\t"                                     \
05609          "sw $4, 24($29) \n\t"                                    \
05610          "lw $4, 32(%1) \n\t"                                     \
05611          "sw $4, 28($29) \n\t"                                    \
05612          "lw $4, 36(%1) \n\t"                                     \
05613          "sw $4, 32($29) \n\t"                                    \
05614          "lw $4, 40(%1) \n\t"                                     \
05615          "sw $4, 36($29) \n\t"                                    \
05616          "lw $4, 4(%1) \n\t"                                      \
05617          "lw $5, 8(%1) \n\t"                                      \
05618          "lw $6, 12(%1) \n\t"                                     \
05619          "lw $7, 16(%1) \n\t"                                     \
05620          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05621          VALGRIND_CALL_NOREDIR_T9                                 \
05622          "addu $29, $29, 48 \n\t"                                 \
05623          "lw $28, 0($29) \n\t"                                    \
05624          "lw $31, 4($29) \n\t"                                    \
05625          "addu $29, $29, 8 \n\t"                                  \
05626          "move %0, $2\n"                                          \
05627          : /*out*/   "=r" (_res)                                  \
05628          : /*in*/    "0" (&_argvec[0])                            \
05629          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05630       );                                                          \
05631       lval = (__typeof__(lval)) _res;                             \
05632    } while (0)
05633 
05634 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
05635                                   arg6,arg7,arg8,arg9,arg10,      \
05636                                   arg11)                          \
05637    do {                                                           \
05638       volatile OrigFn        _orig = (orig);                      \
05639       volatile unsigned long _argvec[12];                         \
05640       volatile unsigned long _res;                                \
05641       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05642       _argvec[1] = (unsigned long)(arg1);                         \
05643       _argvec[2] = (unsigned long)(arg2);                         \
05644       _argvec[3] = (unsigned long)(arg3);                         \
05645       _argvec[4] = (unsigned long)(arg4);                         \
05646       _argvec[5] = (unsigned long)(arg5);                         \
05647       _argvec[6] = (unsigned long)(arg6);                         \
05648       _argvec[7] = (unsigned long)(arg7);                         \
05649       _argvec[8] = (unsigned long)(arg8);                         \
05650       _argvec[9] = (unsigned long)(arg9);                         \
05651       _argvec[10] = (unsigned long)(arg10);                       \
05652       _argvec[11] = (unsigned long)(arg11);                       \
05653       __asm__ volatile(                                           \
05654          "subu $29, $29, 8 \n\t"                                  \
05655          "sw $28, 0($29) \n\t"                                    \
05656          "sw $31, 4($29) \n\t"                                    \
05657          "lw $4, 20(%1) \n\t"                                     \
05658          "subu $29, $29, 48\n\t"                                  \
05659          "sw $4, 16($29) \n\t"                                    \
05660          "lw $4, 24(%1) \n\t"                                     \
05661          "sw $4, 20($29) \n\t"                                    \
05662          "lw $4, 28(%1) \n\t"                                     \
05663          "sw $4, 24($29) \n\t"                                    \
05664          "lw $4, 32(%1) \n\t"                                     \
05665          "sw $4, 28($29) \n\t"                                    \
05666          "lw $4, 36(%1) \n\t"                                     \
05667          "sw $4, 32($29) \n\t"                                    \
05668          "lw $4, 40(%1) \n\t"                                     \
05669          "sw $4, 36($29) \n\t"                                    \
05670          "lw $4, 44(%1) \n\t"                                     \
05671          "sw $4, 40($29) \n\t"                                    \
05672          "lw $4, 4(%1) \n\t"                                      \
05673          "lw $5, 8(%1) \n\t"                                      \
05674          "lw $6, 12(%1) \n\t"                                     \
05675          "lw $7, 16(%1) \n\t"                                     \
05676          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05677          VALGRIND_CALL_NOREDIR_T9                                 \
05678          "addu $29, $29, 48 \n\t"                                 \
05679          "lw $28, 0($29) \n\t"                                    \
05680          "lw $31, 4($29) \n\t"                                    \
05681          "addu $29, $29, 8 \n\t"                                  \
05682          "move %0, $2\n"                                          \
05683          : /*out*/   "=r" (_res)                                  \
05684          : /*in*/    "0" (&_argvec[0])                            \
05685          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05686       );                                                          \
05687       lval = (__typeof__(lval)) _res;                             \
05688    } while (0)
05689 
05690 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
05691                                   arg6,arg7,arg8,arg9,arg10,      \
05692                                   arg11,arg12)                    \
05693    do {                                                           \
05694       volatile OrigFn        _orig = (orig);                      \
05695       volatile unsigned long _argvec[13];                         \
05696       volatile unsigned long _res;                                \
05697       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05698       _argvec[1] = (unsigned long)(arg1);                         \
05699       _argvec[2] = (unsigned long)(arg2);                         \
05700       _argvec[3] = (unsigned long)(arg3);                         \
05701       _argvec[4] = (unsigned long)(arg4);                         \
05702       _argvec[5] = (unsigned long)(arg5);                         \
05703       _argvec[6] = (unsigned long)(arg6);                         \
05704       _argvec[7] = (unsigned long)(arg7);                         \
05705       _argvec[8] = (unsigned long)(arg8);                         \
05706       _argvec[9] = (unsigned long)(arg9);                         \
05707       _argvec[10] = (unsigned long)(arg10);                       \
05708       _argvec[11] = (unsigned long)(arg11);                       \
05709       _argvec[12] = (unsigned long)(arg12);                       \
05710       __asm__ volatile(                                           \
05711          "subu $29, $29, 8 \n\t"                                  \
05712          "sw $28, 0($29) \n\t"                                    \
05713          "sw $31, 4($29) \n\t"                                    \
05714          "lw $4, 20(%1) \n\t"                                     \
05715          "subu $29, $29, 56\n\t"                                  \
05716          "sw $4, 16($29) \n\t"                                    \
05717          "lw $4, 24(%1) \n\t"                                     \
05718          "sw $4, 20($29) \n\t"                                    \
05719          "lw $4, 28(%1) \n\t"                                     \
05720          "sw $4, 24($29) \n\t"                                    \
05721          "lw $4, 32(%1) \n\t"                                     \
05722          "sw $4, 28($29) \n\t"                                    \
05723          "lw $4, 36(%1) \n\t"                                     \
05724          "sw $4, 32($29) \n\t"                                    \
05725          "lw $4, 40(%1) \n\t"                                     \
05726          "sw $4, 36($29) \n\t"                                    \
05727          "lw $4, 44(%1) \n\t"                                     \
05728          "sw $4, 40($29) \n\t"                                    \
05729          "lw $4, 48(%1) \n\t"                                     \
05730          "sw $4, 44($29) \n\t"                                    \
05731          "lw $4, 4(%1) \n\t"                                      \
05732          "lw $5, 8(%1) \n\t"                                      \
05733          "lw $6, 12(%1) \n\t"                                     \
05734          "lw $7, 16(%1) \n\t"                                     \
05735          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
05736          VALGRIND_CALL_NOREDIR_T9                                 \
05737          "addu $29, $29, 56 \n\t"                                 \
05738          "lw $28, 0($29) \n\t"                                    \
05739          "lw $31, 4($29) \n\t"                                    \
05740          "addu $29, $29, 8 \n\t"                                  \
05741          "move %0, $2\n"                                          \
05742          : /*out*/   "=r" (_res)                                  \
05743          : /*in*/    "r" (&_argvec[0])                            \
05744          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05745       );                                                          \
05746       lval = (__typeof__(lval)) _res;                             \
05747    } while (0)
05748 
05749 #endif /* PLAT_mips32_linux */
05750 
05751 /* ------------------------- mips64-linux ------------------------- */
05752 
05753 #if defined(PLAT_mips64_linux)
05754 
05755 /* These regs are trashed by the hidden call. */
05756 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
05757 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
05758 "$25", "$31"
05759 
05760 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
05761    long) == 4. */
05762 
05763 #define CALL_FN_W_v(lval, orig)                                   \
05764    do {                                                           \
05765       volatile OrigFn        _orig = (orig);                      \
05766       volatile unsigned long _argvec[1];                          \
05767       volatile unsigned long _res;                                \
05768       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05769       __asm__ volatile(                                           \
05770          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05771          VALGRIND_CALL_NOREDIR_T9                                 \
05772          "move %0, $2\n"                                          \
05773          : /*out*/   "=r" (_res)                                  \
05774          : /*in*/    "0" (&_argvec[0])                            \
05775          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05776       );                                                          \
05777       lval = (__typeof__(lval)) _res;                             \
05778    } while (0)
05779 
05780 #define CALL_FN_W_W(lval, orig, arg1)                             \
05781    do {                                                           \
05782       volatile OrigFn        _orig = (orig);                      \
05783       volatile unsigned long _argvec[2];                          \
05784       volatile unsigned long _res;                                \
05785       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05786       _argvec[1] = (unsigned long)(arg1);                         \
05787       __asm__ volatile(                                           \
05788          "ld $4, 8(%1)\n\t"   /* arg1*/                           \
05789          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05790          VALGRIND_CALL_NOREDIR_T9                                 \
05791          "move %0, $2\n"                                          \
05792          : /*out*/   "=r" (_res)                                  \
05793          : /*in*/    "r" (&_argvec[0])                            \
05794          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05795       );                                                          \
05796       lval = (__typeof__(lval)) _res;                             \
05797    } while (0)
05798 
05799 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
05800    do {                                                           \
05801       volatile OrigFn        _orig = (orig);                      \
05802       volatile unsigned long _argvec[3];                          \
05803       volatile unsigned long _res;                                \
05804       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05805       _argvec[1] = (unsigned long)(arg1);                         \
05806       _argvec[2] = (unsigned long)(arg2);                         \
05807       __asm__ volatile(                                           \
05808          "ld $4, 8(%1)\n\t"                                       \
05809          "ld $5, 16(%1)\n\t"                                      \
05810          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05811          VALGRIND_CALL_NOREDIR_T9                                 \
05812          "move %0, $2\n"                                          \
05813          : /*out*/   "=r" (_res)                                  \
05814          : /*in*/    "r" (&_argvec[0])                            \
05815          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05816       );                                                          \
05817       lval = (__typeof__(lval)) _res;                             \
05818    } while (0)
05819 
05820 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
05821    do {                                                           \
05822       volatile OrigFn        _orig = (orig);                      \
05823       volatile unsigned long _argvec[4];                          \
05824       volatile unsigned long _res;                                \
05825       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05826       _argvec[1] = (unsigned long)(arg1);                         \
05827       _argvec[2] = (unsigned long)(arg2);                         \
05828       _argvec[3] = (unsigned long)(arg3);                         \
05829       __asm__ volatile(                                           \
05830          "ld $4, 8(%1)\n\t"                                       \
05831          "ld $5, 16(%1)\n\t"                                      \
05832          "ld $6, 24(%1)\n\t"                                      \
05833          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05834          VALGRIND_CALL_NOREDIR_T9                                 \
05835          "move %0, $2\n"                                          \
05836          : /*out*/   "=r" (_res)                                  \
05837          : /*in*/    "r" (&_argvec[0])                            \
05838          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05839       );                                                          \
05840       lval = (__typeof__(lval)) _res;                             \
05841    } while (0)
05842 
05843 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
05844    do {                                                           \
05845       volatile OrigFn        _orig = (orig);                      \
05846       volatile unsigned long _argvec[5];                          \
05847       volatile unsigned long _res;                                \
05848       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05849       _argvec[1] = (unsigned long)(arg1);                         \
05850       _argvec[2] = (unsigned long)(arg2);                         \
05851       _argvec[3] = (unsigned long)(arg3);                         \
05852       _argvec[4] = (unsigned long)(arg4);                         \
05853       __asm__ volatile(                                           \
05854          "ld $4, 8(%1)\n\t"                                       \
05855          "ld $5, 16(%1)\n\t"                                      \
05856          "ld $6, 24(%1)\n\t"                                      \
05857          "ld $7, 32(%1)\n\t"                                      \
05858          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05859          VALGRIND_CALL_NOREDIR_T9                                 \
05860          "move %0, $2\n"                                          \
05861          : /*out*/   "=r" (_res)                                  \
05862          : /*in*/    "r" (&_argvec[0])                            \
05863          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05864       );                                                          \
05865       lval = (__typeof__(lval)) _res;                             \
05866    } while (0)
05867 
05868 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
05869    do {                                                           \
05870       volatile OrigFn        _orig = (orig);                      \
05871       volatile unsigned long _argvec[6];                          \
05872       volatile unsigned long _res;                                \
05873       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05874       _argvec[1] = (unsigned long)(arg1);                         \
05875       _argvec[2] = (unsigned long)(arg2);                         \
05876       _argvec[3] = (unsigned long)(arg3);                         \
05877       _argvec[4] = (unsigned long)(arg4);                         \
05878       _argvec[5] = (unsigned long)(arg5);                         \
05879       __asm__ volatile(                                           \
05880          "ld $4, 8(%1)\n\t"                                       \
05881          "ld $5, 16(%1)\n\t"                                      \
05882          "ld $6, 24(%1)\n\t"                                      \
05883          "ld $7, 32(%1)\n\t"                                      \
05884          "ld $8, 40(%1)\n\t"                                      \
05885          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05886          VALGRIND_CALL_NOREDIR_T9                                 \
05887          "move %0, $2\n"                                          \
05888          : /*out*/   "=r" (_res)                                  \
05889          : /*in*/    "r" (&_argvec[0])                            \
05890          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05891       );                                                          \
05892       lval = (__typeof__(lval)) _res;                             \
05893    } while (0)
05894 
05895 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
05896    do {                                                           \
05897       volatile OrigFn        _orig = (orig);                      \
05898       volatile unsigned long _argvec[7];                          \
05899       volatile unsigned long _res;                                \
05900       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05901       _argvec[1] = (unsigned long)(arg1);                         \
05902       _argvec[2] = (unsigned long)(arg2);                         \
05903       _argvec[3] = (unsigned long)(arg3);                         \
05904       _argvec[4] = (unsigned long)(arg4);                         \
05905       _argvec[5] = (unsigned long)(arg5);                         \
05906       _argvec[6] = (unsigned long)(arg6);                         \
05907       __asm__ volatile(                                           \
05908          "ld $4, 8(%1)\n\t"                                       \
05909          "ld $5, 16(%1)\n\t"                                      \
05910          "ld $6, 24(%1)\n\t"                                      \
05911          "ld $7, 32(%1)\n\t"                                      \
05912          "ld $8, 40(%1)\n\t"                                      \
05913          "ld $9, 48(%1)\n\t"                                      \
05914          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
05915          VALGRIND_CALL_NOREDIR_T9                                 \
05916          "move %0, $2\n"                                          \
05917          : /*out*/   "=r" (_res)                                  \
05918          : /*in*/    "r" (&_argvec[0])                            \
05919          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05920       );                                                          \
05921       lval = (__typeof__(lval)) _res;                             \
05922    } while (0)
05923 
05924 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
05925                                  arg7)                            \
05926    do {                                                           \
05927       volatile OrigFn        _orig = (orig);                      \
05928       volatile unsigned long _argvec[8];                          \
05929       volatile unsigned long _res;                                \
05930       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05931       _argvec[1] = (unsigned long)(arg1);                         \
05932       _argvec[2] = (unsigned long)(arg2);                         \
05933       _argvec[3] = (unsigned long)(arg3);                         \
05934       _argvec[4] = (unsigned long)(arg4);                         \
05935       _argvec[5] = (unsigned long)(arg5);                         \
05936       _argvec[6] = (unsigned long)(arg6);                         \
05937       _argvec[7] = (unsigned long)(arg7);                         \
05938       __asm__ volatile(                                           \
05939          "ld $4, 8(%1)\n\t"                                       \
05940          "ld $5, 16(%1)\n\t"                                      \
05941          "ld $6, 24(%1)\n\t"                                      \
05942          "ld $7, 32(%1)\n\t"                                      \
05943          "ld $8, 40(%1)\n\t"                                      \
05944          "ld $9, 48(%1)\n\t"                                      \
05945          "ld $10, 56(%1)\n\t"                                     \
05946          "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
05947          VALGRIND_CALL_NOREDIR_T9                                 \
05948          "move %0, $2\n"                                          \
05949          : /*out*/   "=r" (_res)                                  \
05950          : /*in*/    "r" (&_argvec[0])                            \
05951          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05952       );                                                          \
05953       lval = (__typeof__(lval)) _res;                             \
05954    } while (0)
05955 
05956 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
05957                                  arg7,arg8)                       \
05958    do {                                                           \
05959       volatile OrigFn        _orig = (orig);                      \
05960       volatile unsigned long _argvec[9];                          \
05961       volatile unsigned long _res;                                \
05962       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05963       _argvec[1] = (unsigned long)(arg1);                         \
05964       _argvec[2] = (unsigned long)(arg2);                         \
05965       _argvec[3] = (unsigned long)(arg3);                         \
05966       _argvec[4] = (unsigned long)(arg4);                         \
05967       _argvec[5] = (unsigned long)(arg5);                         \
05968       _argvec[6] = (unsigned long)(arg6);                         \
05969       _argvec[7] = (unsigned long)(arg7);                         \
05970       _argvec[8] = (unsigned long)(arg8);                         \
05971       __asm__ volatile(                                           \
05972          "ld $4, 8(%1)\n\t"                                       \
05973          "ld $5, 16(%1)\n\t"                                      \
05974          "ld $6, 24(%1)\n\t"                                      \
05975          "ld $7, 32(%1)\n\t"                                      \
05976          "ld $8, 40(%1)\n\t"                                      \
05977          "ld $9, 48(%1)\n\t"                                      \
05978          "ld $10, 56(%1)\n\t"                                     \
05979          "ld $11, 64(%1)\n\t"                                     \
05980          "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
05981          VALGRIND_CALL_NOREDIR_T9                                 \
05982          "move %0, $2\n"                                          \
05983          : /*out*/   "=r" (_res)                                  \
05984          : /*in*/    "r" (&_argvec[0])                            \
05985          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
05986       );                                                          \
05987       lval = (__typeof__(lval)) _res;                             \
05988    } while (0)
05989 
05990 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
05991                                  arg7,arg8,arg9)                  \
05992    do {                                                           \
05993       volatile OrigFn        _orig = (orig);                      \
05994       volatile unsigned long _argvec[10];                         \
05995       volatile unsigned long _res;                                \
05996       _argvec[0] = (unsigned long)_orig.nraddr;                   \
05997       _argvec[1] = (unsigned long)(arg1);                         \
05998       _argvec[2] = (unsigned long)(arg2);                         \
05999       _argvec[3] = (unsigned long)(arg3);                         \
06000       _argvec[4] = (unsigned long)(arg4);                         \
06001       _argvec[5] = (unsigned long)(arg5);                         \
06002       _argvec[6] = (unsigned long)(arg6);                         \
06003       _argvec[7] = (unsigned long)(arg7);                         \
06004       _argvec[8] = (unsigned long)(arg8);                         \
06005       _argvec[9] = (unsigned long)(arg9);                         \
06006       __asm__ volatile(                                           \
06007          "dsubu $29, $29, 8\n\t"                                  \
06008          "ld $4, 72(%1)\n\t"                                      \
06009          "sd $4, 0($29)\n\t"                                      \
06010          "ld $4, 8(%1)\n\t"                                       \
06011          "ld $5, 16(%1)\n\t"                                      \
06012          "ld $6, 24(%1)\n\t"                                      \
06013          "ld $7, 32(%1)\n\t"                                      \
06014          "ld $8, 40(%1)\n\t"                                      \
06015          "ld $9, 48(%1)\n\t"                                      \
06016          "ld $10, 56(%1)\n\t"                                     \
06017          "ld $11, 64(%1)\n\t"                                     \
06018          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
06019          VALGRIND_CALL_NOREDIR_T9                                 \
06020          "daddu $29, $29, 8\n\t"                                  \
06021          "move %0, $2\n"                                          \
06022          : /*out*/   "=r" (_res)                                  \
06023          : /*in*/    "r" (&_argvec[0])                            \
06024          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
06025       );                                                          \
06026       lval = (__typeof__(lval)) _res;                             \
06027    } while (0)
06028 
06029 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
06030                                   arg7,arg8,arg9,arg10)           \
06031    do {                                                           \
06032       volatile OrigFn        _orig = (orig);                      \
06033       volatile unsigned long _argvec[11];                         \
06034       volatile unsigned long _res;                                \
06035       _argvec[0] = (unsigned long)_orig.nraddr;                   \
06036       _argvec[1] = (unsigned long)(arg1);                         \
06037       _argvec[2] = (unsigned long)(arg2);                         \
06038       _argvec[3] = (unsigned long)(arg3);                         \
06039       _argvec[4] = (unsigned long)(arg4);                         \
06040       _argvec[5] = (unsigned long)(arg5);                         \
06041       _argvec[6] = (unsigned long)(arg6);                         \
06042       _argvec[7] = (unsigned long)(arg7);                         \
06043       _argvec[8] = (unsigned long)(arg8);                         \
06044       _argvec[9] = (unsigned long)(arg9);                         \
06045       _argvec[10] = (unsigned long)(arg10);                       \
06046       __asm__ volatile(                                           \
06047          "dsubu $29, $29, 16\n\t"                                 \
06048          "ld $4, 72(%1)\n\t"                                      \
06049          "sd $4, 0($29)\n\t"                                      \
06050          "ld $4, 80(%1)\n\t"                                      \
06051          "sd $4, 8($29)\n\t"                                      \
06052          "ld $4, 8(%1)\n\t"                                       \
06053          "ld $5, 16(%1)\n\t"                                      \
06054          "ld $6, 24(%1)\n\t"                                      \
06055          "ld $7, 32(%1)\n\t"                                      \
06056          "ld $8, 40(%1)\n\t"                                      \
06057          "ld $9, 48(%1)\n\t"                                      \
06058          "ld $10, 56(%1)\n\t"                                     \
06059          "ld $11, 64(%1)\n\t"                                     \
06060          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
06061          VALGRIND_CALL_NOREDIR_T9                                 \
06062          "daddu $29, $29, 16\n\t"                                 \
06063          "move %0, $2\n"                                          \
06064          : /*out*/   "=r" (_res)                                  \
06065          : /*in*/    "r" (&_argvec[0])                            \
06066          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
06067       );                                                          \
06068       lval = (__typeof__(lval)) _res;                             \
06069    } while (0)
06070 
06071 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
06072                                   arg6,arg7,arg8,arg9,arg10,      \
06073                                   arg11)                          \
06074    do {                                                           \
06075       volatile OrigFn        _orig = (orig);                      \
06076       volatile unsigned long _argvec[12];                         \
06077       volatile unsigned long _res;                                \
06078       _argvec[0] = (unsigned long)_orig.nraddr;                   \
06079       _argvec[1] = (unsigned long)(arg1);                         \
06080       _argvec[2] = (unsigned long)(arg2);                         \
06081       _argvec[3] = (unsigned long)(arg3);                         \
06082       _argvec[4] = (unsigned long)(arg4);                         \
06083       _argvec[5] = (unsigned long)(arg5);                         \
06084       _argvec[6] = (unsigned long)(arg6);                         \
06085       _argvec[7] = (unsigned long)(arg7);                         \
06086       _argvec[8] = (unsigned long)(arg8);                         \
06087       _argvec[9] = (unsigned long)(arg9);                         \
06088       _argvec[10] = (unsigned long)(arg10);                       \
06089       _argvec[11] = (unsigned long)(arg11);                       \
06090       __asm__ volatile(                                           \
06091          "dsubu $29, $29, 24\n\t"                                 \
06092          "ld $4, 72(%1)\n\t"                                      \
06093          "sd $4, 0($29)\n\t"                                      \
06094          "ld $4, 80(%1)\n\t"                                      \
06095          "sd $4, 8($29)\n\t"                                      \
06096          "ld $4, 88(%1)\n\t"                                      \
06097          "sd $4, 16($29)\n\t"                                     \
06098          "ld $4, 8(%1)\n\t"                                       \
06099          "ld $5, 16(%1)\n\t"                                      \
06100          "ld $6, 24(%1)\n\t"                                      \
06101          "ld $7, 32(%1)\n\t"                                      \
06102          "ld $8, 40(%1)\n\t"                                      \
06103          "ld $9, 48(%1)\n\t"                                      \
06104          "ld $10, 56(%1)\n\t"                                     \
06105          "ld $11, 64(%1)\n\t"                                     \
06106          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
06107          VALGRIND_CALL_NOREDIR_T9                                 \
06108          "daddu $29, $29, 24\n\t"                                 \
06109          "move %0, $2\n"                                          \
06110          : /*out*/   "=r" (_res)                                  \
06111          : /*in*/    "r" (&_argvec[0])                            \
06112          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
06113       );                                                          \
06114       lval = (__typeof__(lval)) _res;                             \
06115    } while (0)
06116 
06117 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
06118                                   arg6,arg7,arg8,arg9,arg10,      \
06119                                   arg11,arg12)                    \
06120    do {                                                           \
06121       volatile OrigFn        _orig = (orig);                      \
06122       volatile unsigned long _argvec[13];                         \
06123       volatile unsigned long _res;                                \
06124       _argvec[0] = (unsigned long)_orig.nraddr;                   \
06125       _argvec[1] = (unsigned long)(arg1);                         \
06126       _argvec[2] = (unsigned long)(arg2);                         \
06127       _argvec[3] = (unsigned long)(arg3);                         \
06128       _argvec[4] = (unsigned long)(arg4);                         \
06129       _argvec[5] = (unsigned long)(arg5);                         \
06130       _argvec[6] = (unsigned long)(arg6);                         \
06131       _argvec[7] = (unsigned long)(arg7);                         \
06132       _argvec[8] = (unsigned long)(arg8);                         \
06133       _argvec[9] = (unsigned long)(arg9);                         \
06134       _argvec[10] = (unsigned long)(arg10);                       \
06135       _argvec[11] = (unsigned long)(arg11);                       \
06136       _argvec[12] = (unsigned long)(arg12);                       \
06137       __asm__ volatile(                                           \
06138          "dsubu $29, $29, 32\n\t"                                 \
06139          "ld $4, 72(%1)\n\t"                                      \
06140          "sd $4, 0($29)\n\t"                                      \
06141          "ld $4, 80(%1)\n\t"                                      \
06142          "sd $4, 8($29)\n\t"                                      \
06143          "ld $4, 88(%1)\n\t"                                      \
06144          "sd $4, 16($29)\n\t"                                     \
06145          "ld $4, 96(%1)\n\t"                                      \
06146          "sd $4, 24($29)\n\t"                                     \
06147          "ld $4, 8(%1)\n\t"                                       \
06148          "ld $5, 16(%1)\n\t"                                      \
06149          "ld $6, 24(%1)\n\t"                                      \
06150          "ld $7, 32(%1)\n\t"                                      \
06151          "ld $8, 40(%1)\n\t"                                      \
06152          "ld $9, 48(%1)\n\t"                                      \
06153          "ld $10, 56(%1)\n\t"                                     \
06154          "ld $11, 64(%1)\n\t"                                     \
06155          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
06156          VALGRIND_CALL_NOREDIR_T9                                 \
06157          "daddu $29, $29, 32\n\t"                                 \
06158          "move %0, $2\n"                                          \
06159          : /*out*/   "=r" (_res)                                  \
06160          : /*in*/    "r" (&_argvec[0])                            \
06161          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
06162       );                                                          \
06163       lval = (__typeof__(lval)) _res;                             \
06164    } while (0)
06165 
06166 #endif /* PLAT_mips64_linux */
06167 
06168 /* ------------------------ tilegx-linux ------------------------- */
06169 
06170 #if defined(PLAT_tilegx_linux)
06171 
06172 /* These regs are trashed by the hidden call. */
06173 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3", "r4", "r5", \
06174     "r6", "r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14",  \
06175     "r15", "r16", "r17", "r18", "r19", "r20", "r21", "r22",     \
06176     "r23", "r24", "r25", "r26", "r27", "r28", "r29", "lr"
06177 
06178 /* These CALL_FN_ macros assume that on tilegx-linux, sizeof(unsigned
06179    long) == 8. */
06180 
06181 #define CALL_FN_W_v(lval, orig)                          \
06182    do {                                                  \
06183       volatile OrigFn        _orig = (orig);             \
06184       volatile unsigned long _argvec[1];                 \
06185       volatile unsigned long _res;                       \
06186       _argvec[0] = (unsigned long)_orig.nraddr;          \
06187       __asm__ volatile(                                  \
06188          "addi sp, sp, -8 \n\t"                          \
06189          "st_add sp, lr, -8 \n\t"                        \
06190          "ld r12, %1 \n\t"  /* target->r11 */            \
06191          VALGRIND_CALL_NOREDIR_R12                       \
06192          "addi   sp, sp, 8\n\t"                          \
06193          "ld_add lr, sp, 8 \n\t"                         \
06194          "move  %0, r0 \n"                               \
06195          : /*out*/   "=r" (_res)                         \
06196          : /*in*/    "r" (&_argvec[0])                   \
06197          : /*trash*/  "memory", __CALLER_SAVED_REGS);    \
06198                                                          \
06199       lval = (__typeof__(lval)) _res;                    \
06200    } while (0)
06201 
06202 #define CALL_FN_W_W(lval, orig, arg1)                   \
06203    do {                                                 \
06204       volatile OrigFn        _orig = (orig);            \
06205       volatile unsigned long _argvec[2];                \
06206       volatile unsigned long _res;                      \
06207       _argvec[0] = (unsigned long)_orig.nraddr;         \
06208       _argvec[1] = (unsigned long)(arg1);               \
06209       __asm__ volatile(                                 \
06210          "addi sp, sp, -8 \n\t"                         \
06211          "st_add sp, lr, -8 \n\t"                       \
06212          "move r29, %1 \n\t"                            \
06213          "ld_add r12, r29, 8 \n\t"  /* target->r11 */   \
06214          "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */       \
06215          VALGRIND_CALL_NOREDIR_R12                      \
06216          "addi   sp, sp, 8\n\t"                         \
06217          "ld_add lr, sp, 8 \n\t"                        \
06218          "move  %0, r0\n"                               \
06219          : /*out*/   "=r" (_res)                        \
06220          : /*in*/    "r" (&_argvec[0])                  \
06221          : /*trash*/  "memory", __CALLER_SAVED_REGS);   \
06222       lval = (__typeof__(lval)) _res;                   \
06223    } while (0)
06224 
06225 #define CALL_FN_W_WW(lval, orig, arg1,arg2)             \
06226    do {                                                 \
06227       volatile OrigFn        _orig = (orig);            \
06228       volatile unsigned long _argvec[3];                \
06229       volatile unsigned long _res;                      \
06230       _argvec[0] = (unsigned long)_orig.nraddr;         \
06231       _argvec[1] = (unsigned long)(arg1);               \
06232       _argvec[2] = (unsigned long)(arg2);               \
06233       __asm__ volatile(                                 \
06234          "addi sp, sp, -8 \n\t"                         \
06235          "st_add sp, lr, -8 \n\t"                       \
06236          "move r29, %1 \n\t"                            \
06237          "ld_add r12, r29, 8 \n\t"  /* target->r11 */   \
06238          "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */       \
06239          "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */       \
06240          VALGRIND_CALL_NOREDIR_R12                      \
06241          "addi   sp, sp, 8\n\t"                         \
06242          "ld_add lr, sp, 8 \n\t"                        \
06243          "move  %0, r0\n"                               \
06244          : /*out*/   "=r" (_res)                        \
06245          : /*in*/    "r" (&_argvec[0])                  \
06246          : /*trash*/  "memory", __CALLER_SAVED_REGS);   \
06247       lval = (__typeof__(lval)) _res;                   \
06248    } while (0)
06249 
06250 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)       \
06251    do {                                                 \
06252      volatile OrigFn        _orig = (orig);             \
06253      volatile unsigned long _argvec[4];                 \
06254      volatile unsigned long _res;                       \
06255      _argvec[0] = (unsigned long)_orig.nraddr;          \
06256      _argvec[1] = (unsigned long)(arg1);                \
06257      _argvec[2] = (unsigned long)(arg2);                \
06258      _argvec[3] = (unsigned long)(arg3);                \
06259      __asm__ volatile(                                  \
06260         "addi sp, sp, -8 \n\t"                          \
06261         "st_add sp, lr, -8 \n\t"                        \
06262         "move r29, %1 \n\t"                             \
06263         "ld_add r12, r29, 8 \n\t"  /* target->r11 */    \
06264         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */        \
06265         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */        \
06266         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */        \
06267         VALGRIND_CALL_NOREDIR_R12                       \
06268         "addi   sp, sp, 8 \n\t"                         \
06269         "ld_add lr, sp, 8 \n\t"                         \
06270         "move  %0, r0\n"                                \
06271         : /*out*/   "=r" (_res)                         \
06272         : /*in*/    "r" (&_argvec[0])                   \
06273         : /*trash*/  "memory", __CALLER_SAVED_REGS);    \
06274      lval = (__typeof__(lval)) _res;                    \
06275    } while (0)
06276 
06277 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \
06278    do {                                                 \
06279       volatile OrigFn        _orig = (orig);            \
06280       volatile unsigned long _argvec[5];                \
06281       volatile unsigned long _res;                      \
06282       _argvec[0] = (unsigned long)_orig.nraddr;         \
06283       _argvec[1] = (unsigned long)(arg1);               \
06284       _argvec[2] = (unsigned long)(arg2);               \
06285       _argvec[3] = (unsigned long)(arg3);               \
06286       _argvec[4] = (unsigned long)(arg4);               \
06287       __asm__ volatile(                                 \
06288          "addi sp, sp, -8 \n\t"                         \
06289          "st_add sp, lr, -8 \n\t"                       \
06290          "move r29, %1 \n\t"                            \
06291          "ld_add r12, r29, 8 \n\t"  /* target->r11 */   \
06292          "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */       \
06293          "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */       \
06294          "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */       \
06295          "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */       \
06296          VALGRIND_CALL_NOREDIR_R12                      \
06297          "addi   sp, sp, 8\n\t"                         \
06298          "ld_add lr, sp, 8 \n\t"                        \
06299          "move  %0, r0\n"                               \
06300          : /*out*/   "=r" (_res)                        \
06301          : /*in*/    "r" (&_argvec[0])                  \
06302          : /*trash*/  "memory", __CALLER_SAVED_REGS);   \
06303       lval = (__typeof__(lval)) _res;                   \
06304    } while (0)
06305 
06306 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)      \
06307    do {                                                         \
06308       volatile OrigFn        _orig = (orig);                    \
06309       volatile unsigned long _argvec[6];                        \
06310       volatile unsigned long _res;                              \
06311       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06312       _argvec[1] = (unsigned long)(arg1);                       \
06313       _argvec[2] = (unsigned long)(arg2);                       \
06314       _argvec[3] = (unsigned long)(arg3);                       \
06315       _argvec[4] = (unsigned long)(arg4);                       \
06316       _argvec[5] = (unsigned long)(arg5);                       \
06317       __asm__ volatile(                                         \
06318          "addi sp, sp, -8 \n\t"                                 \
06319          "st_add sp, lr, -8 \n\t"                               \
06320          "move r29, %1 \n\t"                                    \
06321          "ld_add r12, r29, 8 \n\t"  /* target->r11 */           \
06322          "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */               \
06323          "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */               \
06324          "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */               \
06325          "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */               \
06326          "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */               \
06327          VALGRIND_CALL_NOREDIR_R12                              \
06328          "addi   sp, sp, 8\n\t"                                 \
06329          "ld_add lr, sp, 8 \n\t"                                \
06330          "move  %0, r0\n"                                       \
06331          : /*out*/   "=r" (_res)                                \
06332          : /*in*/    "r" (&_argvec[0])                          \
06333          : /*trash*/  "memory", __CALLER_SAVED_REGS);           \
06334       lval = (__typeof__(lval)) _res;                           \
06335    } while (0)
06336 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \
06337    do {                                                         \
06338       volatile OrigFn        _orig = (orig);                    \
06339       volatile unsigned long _argvec[7];                        \
06340       volatile unsigned long _res;                              \
06341       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06342       _argvec[1] = (unsigned long)(arg1);                       \
06343       _argvec[2] = (unsigned long)(arg2);                       \
06344       _argvec[3] = (unsigned long)(arg3);                       \
06345       _argvec[4] = (unsigned long)(arg4);                       \
06346       _argvec[5] = (unsigned long)(arg5);                       \
06347       _argvec[6] = (unsigned long)(arg6);                       \
06348       __asm__ volatile(                                         \
06349         "addi sp, sp, -8 \n\t"                                  \
06350         "st_add sp, lr, -8 \n\t"                                \
06351         "move r29, %1 \n\t"                                     \
06352         "ld_add r12, r29, 8 \n\t"  /* target->r11 */            \
06353         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                \
06354         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                \
06355         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                \
06356         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                \
06357         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                \
06358         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                \
06359         VALGRIND_CALL_NOREDIR_R12                               \
06360         "addi   sp, sp, 8\n\t"                                  \
06361         "ld_add lr, sp, 8 \n\t"                                 \
06362         "move  %0, r0\n"                                        \
06363         : /*out*/   "=r" (_res)                                 \
06364         : /*in*/    "r" (&_argvec[0])                           \
06365         : /*trash*/  "memory", __CALLER_SAVED_REGS);            \
06366       lval = (__typeof__(lval)) _res;                           \
06367    } while (0)
06368 
06369 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
06370                      arg7)                                      \
06371    do {                                                         \
06372       volatile OrigFn        _orig = (orig);                    \
06373       volatile unsigned long _argvec[8];                        \
06374       volatile unsigned long _res;                              \
06375       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06376       _argvec[1] = (unsigned long)(arg1);                       \
06377       _argvec[2] = (unsigned long)(arg2);                       \
06378       _argvec[3] = (unsigned long)(arg3);                       \
06379       _argvec[4] = (unsigned long)(arg4);                       \
06380       _argvec[5] = (unsigned long)(arg5);                       \
06381       _argvec[6] = (unsigned long)(arg6);                       \
06382       _argvec[7] = (unsigned long)(arg7);                       \
06383       __asm__ volatile(                                         \
06384         "addi sp, sp, -8 \n\t"                                  \
06385         "st_add sp, lr, -8 \n\t"                                \
06386         "move r29, %1 \n\t"                                     \
06387         "ld_add r12, r29, 8 \n\t"  /* target->r11 */            \
06388         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                \
06389         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                \
06390         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                \
06391         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                \
06392         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                \
06393         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                \
06394         "ld_add r6, r29, 8 \n\t" /*arg7 -> r6 */                \
06395         VALGRIND_CALL_NOREDIR_R12                               \
06396         "addi   sp, sp, 8\n\t"                                  \
06397         "ld_add lr, sp, 8 \n\t"                                 \
06398         "move  %0, r0\n"                                        \
06399         : /*out*/   "=r" (_res)                                 \
06400         : /*in*/    "r" (&_argvec[0])                           \
06401         : /*trash*/  "memory", __CALLER_SAVED_REGS);            \
06402       lval = (__typeof__(lval)) _res;                           \
06403    } while (0)
06404 
06405 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
06406                      arg7,arg8)                                 \
06407    do {                                                         \
06408       volatile OrigFn        _orig = (orig);                    \
06409       volatile unsigned long _argvec[9];                        \
06410       volatile unsigned long _res;                              \
06411       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06412       _argvec[1] = (unsigned long)(arg1);                       \
06413       _argvec[2] = (unsigned long)(arg2);                       \
06414       _argvec[3] = (unsigned long)(arg3);                       \
06415       _argvec[4] = (unsigned long)(arg4);                       \
06416       _argvec[5] = (unsigned long)(arg5);                       \
06417       _argvec[6] = (unsigned long)(arg6);                       \
06418       _argvec[7] = (unsigned long)(arg7);                       \
06419       _argvec[8] = (unsigned long)(arg8);                       \
06420       __asm__ volatile(                                         \
06421         "addi sp, sp, -8 \n\t"                                  \
06422         "st_add sp, lr, -8 \n\t"                                \
06423         "move r29, %1 \n\t"                                     \
06424         "ld_add r12, r29, 8 \n\t"  /* target->r11 */            \
06425         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                \
06426         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                \
06427         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                \
06428         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                \
06429         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                \
06430         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                \
06431         "ld_add r6, r29, 8 \n\t" /*arg7 -> r6 */                \
06432         "ld_add r7, r29, 8 \n\t" /*arg8 -> r7 */                \
06433         VALGRIND_CALL_NOREDIR_R12                               \
06434         "addi   sp, sp, 8\n\t"                                  \
06435         "ld_add lr, sp, 8 \n\t"                                 \
06436         "move  %0, r0\n"                                        \
06437         : /*out*/   "=r" (_res)                                 \
06438         : /*in*/    "r" (&_argvec[0])                           \
06439         : /*trash*/  "memory", __CALLER_SAVED_REGS);            \
06440       lval = (__typeof__(lval)) _res;                           \
06441    } while (0)
06442 
06443 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \
06444                      arg7,arg8,arg9)                            \
06445    do {                                                         \
06446       volatile OrigFn        _orig = (orig);                    \
06447       volatile unsigned long _argvec[10];                       \
06448       volatile unsigned long _res;                              \
06449       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06450       _argvec[1] = (unsigned long)(arg1);                       \
06451       _argvec[2] = (unsigned long)(arg2);                       \
06452       _argvec[3] = (unsigned long)(arg3);                       \
06453       _argvec[4] = (unsigned long)(arg4);                       \
06454       _argvec[5] = (unsigned long)(arg5);                       \
06455       _argvec[6] = (unsigned long)(arg6);                       \
06456       _argvec[7] = (unsigned long)(arg7);                       \
06457       _argvec[8] = (unsigned long)(arg8);                       \
06458       _argvec[9] = (unsigned long)(arg9);                       \
06459       __asm__ volatile(                                         \
06460         "addi sp, sp, -8 \n\t"                                  \
06461         "st_add sp, lr, -8 \n\t"                                \
06462         "move r29, %1 \n\t"                                     \
06463         "ld_add r12, r29, 8 \n\t"  /* target->r11 */            \
06464         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                \
06465         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                \
06466         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                \
06467         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                \
06468         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                \
06469         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                \
06470         "ld_add r6, r29, 8 \n\t" /*arg7 -> r6 */                \
06471         "ld_add r7, r29, 8 \n\t" /*arg8 -> r7 */                \
06472         "ld_add r8, r29, 8 \n\t" /*arg9 -> r8 */                \
06473         VALGRIND_CALL_NOREDIR_R12                               \
06474         "addi   sp, sp, 8\n\t"                                  \
06475         "ld_add lr, sp, 8 \n\t"                                 \
06476         "move  %0, r0\n"                                        \
06477         : /*out*/   "=r" (_res)                                 \
06478         : /*in*/    "r" (&_argvec[0])                           \
06479         : /*trash*/  "memory", __CALLER_SAVED_REGS);            \
06480       lval = (__typeof__(lval)) _res;                           \
06481    } while (0)
06482 
06483 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
06484                       arg7,arg8,arg9,arg10)                             \
06485    do {                                                                 \
06486       volatile OrigFn        _orig = (orig);                            \
06487       volatile unsigned long _argvec[11];                               \
06488       volatile unsigned long _res;                                      \
06489       _argvec[0] = (unsigned long)_orig.nraddr;                         \
06490       _argvec[1] = (unsigned long)(arg1);                               \
06491       _argvec[2] = (unsigned long)(arg2);                               \
06492       _argvec[3] = (unsigned long)(arg3);                               \
06493       _argvec[4] = (unsigned long)(arg4);                               \
06494       _argvec[5] = (unsigned long)(arg5);                               \
06495       _argvec[6] = (unsigned long)(arg6);                               \
06496       _argvec[7] = (unsigned long)(arg7);                               \
06497       _argvec[8] = (unsigned long)(arg8);                               \
06498       _argvec[9] = (unsigned long)(arg9);                               \
06499       _argvec[10] = (unsigned long)(arg10);                             \
06500       __asm__ volatile(                                                 \
06501         "addi sp, sp, -8 \n\t"                                          \
06502         "st_add sp, lr, -8 \n\t"                                        \
06503         "move r29, %1 \n\t"                                             \
06504         "ld_add r12, r29, 8 \n\t"  /* target->r11 */                    \
06505         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                        \
06506         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                        \
06507         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                        \
06508         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                        \
06509         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                        \
06510         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                        \
06511         "ld_add r6, r29, 8 \n\t" /*arg7 -> r6 */                        \
06512         "ld_add r7, r29, 8 \n\t" /*arg8 -> r7 */                        \
06513         "ld_add r8, r29, 8 \n\t" /*arg9 -> r8 */                        \
06514         "ld_add r9, r29, 8 \n\t" /*arg10 -> r9 */                       \
06515         VALGRIND_CALL_NOREDIR_R12                                       \
06516         "addi   sp, sp, 8\n\t"                                          \
06517         "ld_add lr, sp, 8 \n\t"                                         \
06518         "move  %0, r0\n"                                                \
06519         : /*out*/   "=r" (_res)                                         \
06520         : /*in*/    "r" (&_argvec[0])                                   \
06521         : /*trash*/  "memory", __CALLER_SAVED_REGS);                    \
06522       lval = (__typeof__(lval)) _res;                                   \
06523    } while (0)
06524 
06525 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,     \
06526                       arg6,arg7,arg8,arg9,arg10,                \
06527                       arg11)                                    \
06528    do {                                                         \
06529       volatile OrigFn        _orig = (orig);                    \
06530       volatile unsigned long _argvec[12];                       \
06531       volatile unsigned long _res;                              \
06532       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06533       _argvec[1] = (unsigned long)(arg1);                       \
06534       _argvec[2] = (unsigned long)(arg2);                       \
06535       _argvec[3] = (unsigned long)(arg3);                       \
06536       _argvec[4] = (unsigned long)(arg4);                       \
06537       _argvec[5] = (unsigned long)(arg5);                       \
06538       _argvec[6] = (unsigned long)(arg6);                       \
06539       _argvec[7] = (unsigned long)(arg7);                       \
06540       _argvec[8] = (unsigned long)(arg8);                       \
06541       _argvec[9] = (unsigned long)(arg9);                       \
06542       _argvec[10] = (unsigned long)(arg10);                     \
06543       _argvec[11] = (unsigned long)(arg11);                     \
06544       __asm__ volatile(                                         \
06545         "addi sp, sp, -8 \n\t"                                  \
06546         "st_add sp, lr, -8 \n\t"                                \
06547         "move r29, %1 \n\t"                                     \
06548         "ld_add r12, r29, 8 \n\t"  /* target->r11 */            \
06549         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                \
06550         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                \
06551         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                \
06552         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                \
06553         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                \
06554         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                \
06555         "ld_add r6, r29, 8 \n\t" /*arg7 -> r6 */                \
06556         "ld_add r7, r29, 8 \n\t" /*arg8 -> r7 */                \
06557         "ld_add r8, r29, 8 \n\t" /*arg9 -> r8 */                \
06558         "ld_add r9, r29, 8 \n\t" /*arg10 -> r9 */               \
06559         "ld     r10, r29 \n\t"                                  \
06560         "st_add sp, r10, -16 \n\t"                              \
06561         VALGRIND_CALL_NOREDIR_R12                               \
06562         "addi   sp, sp, 24 \n\t"                                \
06563         "ld_add lr, sp, 8 \n\t"                                 \
06564         "move  %0, r0\n"                                        \
06565         : /*out*/   "=r" (_res)                                 \
06566         : /*in*/    "r" (&_argvec[0])                           \
06567         : /*trash*/  "memory", __CALLER_SAVED_REGS);            \
06568       lval = (__typeof__(lval)) _res;                           \
06569    } while (0)
06570 
06571 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,     \
06572                       arg6,arg7,arg8,arg9,arg10,                \
06573                       arg11,arg12)                              \
06574    do {                                                         \
06575       volatile OrigFn        _orig = (orig);                    \
06576       volatile unsigned long _argvec[13];                       \
06577       volatile unsigned long _res;                              \
06578       _argvec[0] = (unsigned long)_orig.nraddr;                 \
06579       _argvec[1] = (unsigned long)(arg1);                       \
06580       _argvec[2] = (unsigned long)(arg2);                       \
06581       _argvec[3] = (unsigned long)(arg3);                       \
06582       _argvec[4] = (unsigned long)(arg4);                       \
06583       _argvec[5] = (unsigned long)(arg5);                       \
06584       _argvec[6] = (unsigned long)(arg6);                       \
06585       _argvec[7] = (unsigned long)(arg7);                       \
06586       _argvec[8] = (unsigned long)(arg8);                       \
06587       _argvec[9] = (unsigned long)(arg9);                       \
06588       _argvec[10] = (unsigned long)(arg10);                     \
06589       _argvec[11] = (unsigned long)(arg11);                     \
06590       _argvec[12] = (unsigned long)(arg12);                     \
06591       __asm__ volatile(                                         \
06592         "addi sp, sp, -8 \n\t"                                  \
06593         "st_add sp, lr, -8 \n\t"                                \
06594         "move r29, %1 \n\t"                                     \
06595         "ld_add r12, r29, 8 \n\t"  /* target->r11 */            \
06596         "ld_add r0, r29, 8 \n\t" /*arg1 -> r0 */                \
06597         "ld_add r1, r29, 8 \n\t" /*arg2 -> r1 */                \
06598         "ld_add r2, r29, 8 \n\t" /*arg3 -> r2 */                \
06599         "ld_add r3, r29, 8 \n\t" /*arg4 -> r3 */                \
06600         "ld_add r4, r29, 8 \n\t" /*arg5 -> r4 */                \
06601         "ld_add r5, r29, 8 \n\t" /*arg6 -> r5 */                \
06602         "ld_add r6, r29, 8 \n\t" /*arg7 -> r6 */                \
06603         "ld_add r7, r29, 8 \n\t" /*arg8 -> r7 */                \
06604         "ld_add r8, r29, 8 \n\t" /*arg9 -> r8 */                \
06605         "ld_add r9, r29, 8 \n\t" /*arg10 -> r9 */               \
06606         "addi r28, sp, -8 \n\t"                                 \
06607         "addi sp,  sp, -24 \n\t"                                \
06608         "ld_add r10, r29, 8 \n\t"                               \
06609         "ld     r11, r29 \n\t"                                  \
06610         "st_add r28, r10, 8 \n\t"                               \
06611         "st     r28, r11 \n\t"                                  \
06612         VALGRIND_CALL_NOREDIR_R12                               \
06613         "addi   sp, sp, 32 \n\t"                                \
06614         "ld_add lr, sp, 8 \n\t"                                 \
06615         "move  %0, r0\n"                                        \
06616         : /*out*/   "=r" (_res)                                 \
06617         : /*in*/    "r" (&_argvec[0])                           \
06618         : /*trash*/  "memory", __CALLER_SAVED_REGS);            \
06619       lval = (__typeof__(lval)) _res;                           \
06620    } while (0)
06621 #endif  /* PLAT_tilegx_linux */
06622 
06623 /* ------------------------------------------------------------------ */
06624 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
06625 /*                                                                    */
06626 /* ------------------------------------------------------------------ */
06627 
06628 /* Some request codes.  There are many more of these, but most are not
06629    exposed to end-user view.  These are the public ones, all of the
06630    form 0x1000 + small_number.
06631 
06632    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
06633    ones start at 0x2000.
06634 */
06635 
06636 /* These macros are used by tools -- they must be public, but don't
06637    embed them into other programs. */
06638 #define VG_USERREQ_TOOL_BASE(a,b) \
06639    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
06640 #define VG_IS_TOOL_USERREQ(a, b, v) \
06641    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
06642 
06643 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! 
06644    This enum comprises an ABI exported by Valgrind to programs
06645    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
06646    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
06647 typedef
06648    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
06649           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
06650 
06651           /* These allow any function to be called from the simulated
06652              CPU but run on the real CPU.  Nb: the first arg passed to
06653              the function is always the ThreadId of the running
06654              thread!  So CLIENT_CALL0 actually requires a 1 arg
06655              function, etc. */
06656           VG_USERREQ__CLIENT_CALL0 = 0x1101,
06657           VG_USERREQ__CLIENT_CALL1 = 0x1102,
06658           VG_USERREQ__CLIENT_CALL2 = 0x1103,
06659           VG_USERREQ__CLIENT_CALL3 = 0x1104,
06660 
06661           /* Can be useful in regression testing suites -- eg. can
06662              send Valgrind's output to /dev/null and still count
06663              errors. */
06664           VG_USERREQ__COUNT_ERRORS = 0x1201,
06665 
06666           /* Allows the client program and/or gdbserver to execute a monitor
06667              command. */
06668           VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202,
06669 
06670           /* These are useful and can be interpreted by any tool that
06671              tracks malloc() et al, by using vg_replace_malloc.c. */
06672           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
06673           VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b,
06674           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
06675           /* Memory pool support. */
06676           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
06677           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
06678           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
06679           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
06680           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
06681           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
06682           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
06683           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
06684 
06685           /* Allow printfs to valgrind log. */
06686           /* The first two pass the va_list argument by value, which
06687              assumes it is the same size as or smaller than a UWord,
06688              which generally isn't the case.  Hence are deprecated.
06689              The second two pass the vargs by reference and so are
06690              immune to this problem. */
06691           /* both :: char* fmt, va_list vargs (DEPRECATED) */
06692           VG_USERREQ__PRINTF           = 0x1401,
06693           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
06694           /* both :: char* fmt, va_list* vargs */
06695           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
06696           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
06697 
06698           /* Stack support. */
06699           VG_USERREQ__STACK_REGISTER   = 0x1501,
06700           VG_USERREQ__STACK_DEREGISTER = 0x1502,
06701           VG_USERREQ__STACK_CHANGE     = 0x1503,
06702 
06703           /* Wine support */
06704           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
06705 
06706           /* Querying of debug info. */
06707           VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701,
06708 
06709           /* Disable/enable error reporting level.  Takes a single
06710              Word arg which is the delta to this thread's error
06711              disablement indicator.  Hence 1 disables or further
06712              disables errors, and -1 moves back towards enablement.
06713              Other values are not allowed. */
06714           VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801,
06715 
06716           /* Initialise IR injection */
06717           VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901
06718    } Vg_ClientRequest;
06719 
06720 #if !defined(__GNUC__)
06721 #  define __extension__ /* */
06722 #endif
06723 
06724 
06725 /* Returns the number of Valgrinds this code is running under.  That
06726    is, 0 if running natively, 1 if running under Valgrind, 2 if
06727    running under Valgrind which is running under another Valgrind,
06728    etc. */
06729 #define RUNNING_ON_VALGRIND                                           \
06730     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,         \
06731                                     VG_USERREQ__RUNNING_ON_VALGRIND,  \
06732                                     0, 0, 0, 0, 0)                    \
06733 
06734 
06735 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
06736    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
06737    since it provides a way to make sure valgrind will retranslate the
06738    invalidated area.  Returns no value. */
06739 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)              \
06740     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS,  \
06741                                     _qzz_addr, _qzz_len, 0, 0, 0)
06742 
06743 
06744 /* These requests are for getting Valgrind itself to print something.
06745    Possibly with a backtrace.  This is a really ugly hack.  The return value
06746    is the number of characters printed, excluding the "**<pid>** " part at the
06747    start and the backtrace (if present). */
06748 
06749 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
06750 /* Modern GCC will optimize the static routine out if unused,
06751    and unused attribute will shut down warnings about it.  */
06752 static int VALGRIND_PRINTF(const char *format, ...)
06753    __attribute__((format(__printf__, 1, 2), __unused__));
06754 #endif
06755 static int
06756 #if defined(_MSC_VER)
06757 __inline
06758 #endif
06759 VALGRIND_PRINTF(const char *format, ...)
06760 {
06761 #if defined(NVALGRIND)
06762    return 0;
06763 #else /* NVALGRIND */
06764 #if defined(_MSC_VER) || defined(__MINGW64__)
06765    uintptr_t _qzz_res;
06766 #else
06767    unsigned long _qzz_res;
06768 #endif
06769    va_list vargs;
06770    va_start(vargs, format);
06771 #if defined(_MSC_VER) || defined(__MINGW64__)
06772    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
06773                               VG_USERREQ__PRINTF_VALIST_BY_REF,
06774                               (uintptr_t)format,
06775                               (uintptr_t)&vargs,
06776                               0, 0, 0);
06777 #else
06778    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
06779                               VG_USERREQ__PRINTF_VALIST_BY_REF,
06780                               (unsigned long)format,
06781                               (unsigned long)&vargs, 
06782                               0, 0, 0);
06783 #endif
06784    va_end(vargs);
06785    return (int)_qzz_res;
06786 #endif /* NVALGRIND */
06787 }
06788 
06789 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
06790 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
06791    __attribute__((format(__printf__, 1, 2), __unused__));
06792 #endif
06793 static int
06794 #if defined(_MSC_VER)
06795 __inline
06796 #endif
06797 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
06798 {
06799 #if defined(NVALGRIND)
06800    return 0;
06801 #else /* NVALGRIND */
06802 #if defined(_MSC_VER) || defined(__MINGW64__)
06803    uintptr_t _qzz_res;
06804 #else
06805    unsigned long _qzz_res;
06806 #endif
06807    va_list vargs;
06808    va_start(vargs, format);
06809 #if defined(_MSC_VER) || defined(__MINGW64__)
06810    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
06811                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
06812                               (uintptr_t)format,
06813                               (uintptr_t)&vargs,
06814                               0, 0, 0);
06815 #else
06816    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
06817                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
06818                               (unsigned long)format,
06819                               (unsigned long)&vargs, 
06820                               0, 0, 0);
06821 #endif
06822    va_end(vargs);
06823    return (int)_qzz_res;
06824 #endif /* NVALGRIND */
06825 }
06826 
06827 
06828 /* These requests allow control to move from the simulated CPU to the
06829    real CPU, calling an arbitary function.
06830    
06831    Note that the current ThreadId is inserted as the first argument.
06832    So this call:
06833 
06834      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
06835 
06836    requires f to have this signature:
06837 
06838      Word f(Word tid, Word arg1, Word arg2)
06839 
06840    where "Word" is a word-sized type.
06841 
06842    Note that these client requests are not entirely reliable.  For example,
06843    if you call a function with them that subsequently calls printf(),
06844    there's a high chance Valgrind will crash.  Generally, your prospects of
06845    these working are made higher if the called function does not refer to
06846    any global variables, and does not refer to any libc or other functions
06847    (printf et al).  Any kind of entanglement with libc or dynamic linking is
06848    likely to have a bad outcome, for tricky reasons which we've grappled
06849    with a lot in the past.
06850 */
06851 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
06852     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,       \
06853                                     VG_USERREQ__CLIENT_CALL0,     \
06854                                     _qyy_fn,                      \
06855                                     0, 0, 0, 0)
06856 
06857 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)                    \
06858     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
06859                                     VG_USERREQ__CLIENT_CALL1,          \
06860                                     _qyy_fn,                           \
06861                                     _qyy_arg1, 0, 0, 0)
06862 
06863 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)         \
06864     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
06865                                     VG_USERREQ__CLIENT_CALL2,          \
06866                                     _qyy_fn,                           \
06867                                     _qyy_arg1, _qyy_arg2, 0, 0)
06868 
06869 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
06870     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,             \
06871                                     VG_USERREQ__CLIENT_CALL3,           \
06872                                     _qyy_fn,                            \
06873                                     _qyy_arg1, _qyy_arg2,               \
06874                                     _qyy_arg3, 0)
06875 
06876 
06877 /* Counts the number of errors that have been recorded by a tool.  Nb:
06878    the tool must record the errors with VG_(maybe_record_error)() or
06879    VG_(unique_error)() for them to be counted. */
06880 #define VALGRIND_COUNT_ERRORS                                     \
06881     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(                    \
06882                                0 /* default return */,            \
06883                                VG_USERREQ__COUNT_ERRORS,          \
06884                                0, 0, 0, 0, 0)
06885 
06886 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
06887    when heap blocks are allocated in order to give accurate results.  This
06888    happens automatically for the standard allocator functions such as
06889    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
06890    delete[], etc.
06891 
06892    But if your program uses a custom allocator, this doesn't automatically
06893    happen, and Valgrind will not do as well.  For example, if you allocate
06894    superblocks with mmap() and then allocates chunks of the superblocks, all
06895    Valgrind's observations will be at the mmap() level and it won't know that
06896    the chunks should be considered separate entities.  In Memcheck's case,
06897    that means you probably won't get heap block overrun detection (because
06898    there won't be redzones marked as unaddressable) and you definitely won't
06899    get any leak detection.
06900 
06901    The following client requests allow a custom allocator to be annotated so
06902    that it can be handled accurately by Valgrind.
06903 
06904    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
06905    by a malloc()-like function.  For Memcheck (an illustrative case), this
06906    does two things:
06907 
06908    - It records that the block has been allocated.  This means any addresses
06909      within the block mentioned in error messages will be
06910      identified as belonging to the block.  It also means that if the block
06911      isn't freed it will be detected by the leak checker.
06912 
06913    - It marks the block as being addressable and undefined (if 'is_zeroed' is
06914      not set), or addressable and defined (if 'is_zeroed' is set).  This
06915      controls how accesses to the block by the program are handled.
06916    
06917    'addr' is the start of the usable block (ie. after any
06918    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
06919    can apply redzones -- these are blocks of padding at the start and end of
06920    each block.  Adding redzones is recommended as it makes it much more likely
06921    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
06922    zeroed (or filled with another predictable value), as is the case for
06923    calloc().
06924    
06925    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
06926    heap block -- that will be used by the client program -- is allocated.
06927    It's best to put it at the outermost level of the allocator if possible;
06928    for example, if you have a function my_alloc() which calls
06929    internal_alloc(), and the client request is put inside internal_alloc(),
06930    stack traces relating to the heap block will contain entries for both
06931    my_alloc() and internal_alloc(), which is probably not what you want.
06932 
06933    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
06934    custom blocks from within a heap block, B, that has been allocated with
06935    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
06936    -- the custom blocks will take precedence.
06937 
06938    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
06939    Memcheck, it does two things:
06940 
06941    - It records that the block has been deallocated.  This assumes that the
06942      block was annotated as having been allocated via
06943      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
06944 
06945    - It marks the block as being unaddressable.
06946 
06947    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
06948    heap block is deallocated.
06949 
06950    VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For
06951    Memcheck, it does four things:
06952 
06953    - It records that the size of a block has been changed.  This assumes that
06954      the block was annotated as having been allocated via
06955      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
06956 
06957    - If the block shrunk, it marks the freed memory as being unaddressable.
06958 
06959    - If the block grew, it marks the new area as undefined and defines a red
06960      zone past the end of the new block.
06961 
06962    - The V-bits of the overlap between the old and the new block are preserved.
06963 
06964    VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block
06965    and before deallocation of the old block.
06966 
06967    In many cases, these three client requests will not be enough to get your
06968    allocator working well with Memcheck.  More specifically, if your allocator
06969    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
06970    will be necessary to mark the memory as addressable just before the zeroing
06971    occurs, otherwise you'll get a lot of invalid write errors.  For example,
06972    you'll need to do this if your allocator recycles freed blocks, but it
06973    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
06974    Alternatively, if your allocator reuses freed blocks for allocator-internal
06975    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
06976 
06977    Really, what's happening is a blurring of the lines between the client
06978    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
06979    memory should be considered unaddressable to the client program, but the
06980    allocator knows more than the rest of the client program and so may be able
06981    to safely access it.  Extra client requests are necessary for Valgrind to
06982    understand the distinction between the allocator and the rest of the
06983    program.
06984 
06985    Ignored if addr == 0.
06986 */
06987 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)          \
06988     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK,       \
06989                                     addr, sizeB, rzB, is_zeroed, 0)
06990 
06991 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
06992    Ignored if addr == 0.
06993 */
06994 #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)     \
06995     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK,    \
06996                                     addr, oldSizeB, newSizeB, rzB, 0)
06997 
06998 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
06999    Ignored if addr == 0.
07000 */
07001 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                              \
07002     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK,         \
07003                                     addr, rzB, 0, 0, 0)
07004 
07005 /* Create a memory pool. */
07006 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
07007     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL,   \
07008                                     pool, rzB, is_zeroed, 0, 0)
07009 
07010 /* Destroy a memory pool. */
07011 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
07012     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL,  \
07013                                     pool, 0, 0, 0, 0)
07014 
07015 /* Associate a piece of memory with a memory pool. */
07016 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
07017     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC,    \
07018                                     pool, addr, size, 0, 0)
07019 
07020 /* Disassociate a piece of memory from a memory pool. */
07021 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
07022     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE,     \
07023                                     pool, addr, 0, 0, 0)
07024 
07025 /* Disassociate any pieces outside a particular range. */
07026 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
07027     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM,     \
07028                                     pool, addr, size, 0, 0)
07029 
07030 /* Resize and/or move a piece associated with a memory pool. */
07031 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
07032     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL,     \
07033                                     poolA, poolB, 0, 0, 0)
07034 
07035 /* Resize and/or move a piece associated with a memory pool. */
07036 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
07037     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE,   \
07038                                     pool, addrA, addrB, size, 0)
07039 
07040 /* Return 1 if a mempool exists, else 0. */
07041 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
07042     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
07043                                VG_USERREQ__MEMPOOL_EXISTS,        \
07044                                pool, 0, 0, 0, 0)
07045 
07046 /* Mark a piece of memory as being a stack. Returns a stack id.
07047    start is the lowest addressable stack byte, end is the highest
07048    addressable stack byte. */
07049 #define VALGRIND_STACK_REGISTER(start, end)                       \
07050     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
07051                                VG_USERREQ__STACK_REGISTER,        \
07052                                start, end, 0, 0, 0)
07053 
07054 /* Unmark the piece of memory associated with a stack id as being a
07055    stack. */
07056 #define VALGRIND_STACK_DEREGISTER(id)                             \
07057     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \
07058                                     id, 0, 0, 0, 0)
07059 
07060 /* Change the start and end address of the stack id.
07061    start is the new lowest addressable stack byte, end is the new highest
07062    addressable stack byte. */
07063 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
07064     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE,     \
07065                                     id, start, end, 0, 0)
07066 
07067 /* Load PDB debug info for Wine PE image_map. */
07068 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)     \
07069     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \
07070                                     fd, ptr, total_size, delta, 0)
07071 
07072 /* Map a code address to a source file name and line number.  buf64
07073    must point to a 64-byte buffer in the caller's address space.  The
07074    result will be dumped in there and is guaranteed to be zero
07075    terminated.  If no info is found, the first byte is set to zero. */
07076 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
07077     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
07078                                VG_USERREQ__MAP_IP_TO_SRCLOC,      \
07079                                addr, buf64, 0, 0, 0)
07080 
07081 /* Disable error reporting for this thread.  Behaves in a stack like
07082    way, so you can safely call this multiple times provided that
07083    VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times
07084    to re-enable reporting.  The first call of this macro disables
07085    reporting.  Subsequent calls have no effect except to increase the
07086    number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable
07087    reporting.  Child threads do not inherit this setting from their
07088    parents -- they are always created with reporting enabled. */
07089 #define VALGRIND_DISABLE_ERROR_REPORTING                                \
07090     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
07091                                     1, 0, 0, 0, 0)
07092 
07093 /* Re-enable error reporting, as per comments on
07094    VALGRIND_DISABLE_ERROR_REPORTING. */
07095 #define VALGRIND_ENABLE_ERROR_REPORTING                                 \
07096     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
07097                                     -1, 0, 0, 0, 0)
07098 
07099 /* Execute a monitor command from the client program.
07100    If a connection is opened with GDB, the output will be sent
07101    according to the output mode set for vgdb.
07102    If no connection is opened, output will go to the log output.
07103    Returns 1 if command not recognised, 0 otherwise. */
07104 #define VALGRIND_MONITOR_COMMAND(command)                               \
07105    VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \
07106                                    command, 0, 0, 0, 0)
07107 
07108 
07109 #undef PLAT_x86_darwin
07110 #undef PLAT_amd64_darwin
07111 #undef PLAT_x86_win32
07112 #undef PLAT_amd64_win64
07113 #undef PLAT_x86_linux
07114 #undef PLAT_amd64_linux
07115 #undef PLAT_ppc32_linux
07116 #undef PLAT_ppc64be_linux
07117 #undef PLAT_ppc64le_linux
07118 #undef PLAT_arm_linux
07119 #undef PLAT_s390x_linux
07120 #undef PLAT_mips32_linux
07121 #undef PLAT_mips64_linux
07122 #undef PLAT_tilegx_linux
07123 #undef PLAT_x86_solaris
07124 #undef PLAT_amd64_solaris
07125 
07126 #endif   /* __VALGRIND_H */
util/valgrind.h
