1 /*

     2  * $Id: ptmalloc3.c,v 1.8 2006/03/31 15:57:28 wg Exp $

     3  * 

     4 

     5 ptmalloc3 -- wrapper for Doug Lea's malloc-2.8.3 with concurrent

     6              allocations

     7 

     8 Copyright (c) 2005, 2006 Wolfram Gloger  <ptmalloc@malloc.de>

     9 

    10 Permission to use, copy, modify, distribute, and sell this software

    11 and its documentation for any purpose is hereby granted without fee,

    12 provided that (i) the above copyright notices and this permission

    13 notice appear in all copies of the software and related documentation,

    14 and (ii) the name of Wolfram Gloger may not be used in any advertising

    15 or publicity relating to the software.

    16 

    17 THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND,

    18 EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY

    19 WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.

    20 

    21 IN NO EVENT SHALL WOLFRAM GLOGER BE LIABLE FOR ANY SPECIAL,

    22 INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND, OR ANY

    23 DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,

    24 WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY

    25 OF LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR

    26 PERFORMANCE OF THIS SOFTWARE.

    27 

    28  */

    29 

    30 /*

    31  * TODO: optimization / better integration with malloc.c (partly done)

    32  *       malloc_{get,set}_state (probably hard to keep compatibility)

    33  *       debugging hooks

    34  *       better mstats

    35  */

    36 

    37 #include <sys/types.h>   /* For size_t */

    38 #include <sys/mman.h>    /* for mmap */

    39 #include <errno.h>

    40 #include <stdlib.h>

    41 #include <stdio.h>

    42 #include <string.h>      /* for memset */

    43 

    44 #include <malloc-machine.h>

    45 

    46 #include "malloc-2.8.3.h"

    47 

    48 /* ----------------------------------------------------------------------- */

    49 

    50 /* The following section is replicated from malloc.c */

    51 

    52 #include "malloc-private.h"

    53 

    54 /* end of definitions replicated from malloc.c */

    55 

    56 #define munmap_chunk(mst, p) do {                         \

    57   size_t prevsize = (p)->prev_foot & ~IS_MMAPPED_BIT;     \

    58   size_t psize = chunksize(p) + prevsize + MMAP_FOOT_PAD; \

    59   if (CALL_MUNMAP((char*)(p) - prevsize, psize) == 0)     \

    60     ((struct malloc_state*)(mst))->footprint -= psize;    \

    61 } while (0)

    62 

    63 /* ---------------------------------------------------------------------- */

    64 

    65 /* Minimum size for a newly created arena.  */

    66 #ifndef ARENA_SIZE_MIN

    67 # define ARENA_SIZE_MIN	   (128*1024)

    68 #endif

    69 #define HAVE_MEMCPY        1

    70 

    71 /* If THREAD_STATS is non-zero, some statistics on mutex locking are

    72    computed.  */

    73 #ifndef THREAD_STATS

    74 # define THREAD_STATS 0

    75 #endif

    76 

    77 #ifndef MALLOC_DEBUG

    78 # define MALLOC_DEBUG 0

    79 #endif

    80 

    81 #define my_powerof2(x) ((((x)-1)&(x))==0)

    82 

    83 /* Already initialized? */

    84 int __malloc_initialized = -1;

    85 

    86 #ifndef RETURN_ADDRESS

    87 # define RETURN_ADDRESS(X_) (NULL)

    88 #endif

    89 

    90 #if THREAD_STATS

    91 # define THREAD_STAT(x) x

    92 #else

    93 # define THREAD_STAT(x) do ; while(0)

    94 #endif

    95 

    96 #ifdef _LIBC

    97 

    98 /* Special defines for the GNU C library.  */

    99 #define public_cALLOc    __libc_calloc

   100 #define public_fREe      __libc_free

   101 #define public_cFREe     __libc_cfree

   102 #define public_mALLOc    __libc_malloc

   103 #define public_mEMALIGn  __libc_memalign

   104 #define public_rEALLOc   __libc_realloc

   105 #define public_vALLOc    __libc_valloc

   106 #define public_pVALLOc   __libc_pvalloc

   107 #define public_pMEMALIGn __posix_memalign

   108 #define public_mALLINFo  __libc_mallinfo

   109 #define public_mALLOPt   __libc_mallopt

   110 #define public_mTRIm     __malloc_trim

   111 #define public_mSTATs    __malloc_stats

   112 #define public_mUSABLe   __malloc_usable_size

   113 #define public_iCALLOc   __libc_independent_calloc

   114 #define public_iCOMALLOc __libc_independent_comalloc

   115 #define public_gET_STATe __malloc_get_state

   116 #define public_sET_STATe __malloc_set_state

   117 #define malloc_getpagesize __getpagesize()

   118 #define open             __open

   119 #define mmap             __mmap

   120 #define munmap           __munmap

   121 #define mremap           __mremap

   122 #define mprotect         __mprotect

   123 #define MORECORE         (*__morecore)

   124 #define MORECORE_FAILURE 0

   125 

   126 void * __default_morecore (ptrdiff_t);

   127 void *(*__morecore)(ptrdiff_t) = __default_morecore;

   128 

   129 #else /* !_LIBC */

   130 

   131 #define public_cALLOc    calloc

   132 #define public_fREe      free

   133 #define public_cFREe     cfree

   134 #define public_mALLOc    malloc

   135 #define public_mEMALIGn  memalign

   136 #define public_rEALLOc   realloc

   137 #define public_vALLOc    valloc

   138 #define public_pVALLOc   pvalloc

   139 #define public_pMEMALIGn posix_memalign

   140 #define public_mALLINFo  mallinfo

   141 #define public_mALLOPt   mallopt

   142 #define public_mTRIm     malloc_trim

   143 #define public_mSTATs    malloc_stats

   144 #define public_mUSABLe   malloc_usable_size

   145 #define public_iCALLOc   independent_calloc

   146 #define public_iCOMALLOc independent_comalloc

   147 #define public_gET_STATe malloc_get_state

   148 #define public_sET_STATe malloc_set_state

   149 

   150 #endif /* _LIBC */

   151 

   152 #if !defined _LIBC && (!defined __GNUC__ || __GNUC__<3)

   153 #define __builtin_expect(expr, val) (expr)

   154 #endif

   155 

   156 #if MALLOC_DEBUG

   157 #include <assert.h>

   158 #else

   159 #undef assert

   160 #define assert(x) ((void)0)

   161 #endif

   162 

   163 /* USE_STARTER determines if and when the special "starter" hook

   164    functions are used: not at all (0), during ptmalloc_init (first bit

   165    set), or from the beginning until an explicit call to ptmalloc_init

   166    (second bit set).  This is necessary if thread-related

   167    initialization functions (e.g.  pthread_key_create) require

   168    malloc() calls (set USE_STARTER=1), or if those functions initially

   169    cannot be used at all (set USE_STARTER=2 and perform an explicit

   170    ptmalloc_init() when the thread library is ready, typically at the

   171    start of main()). */

   172 

   173 #ifndef USE_STARTER

   174 # ifndef _LIBC

   175 #  define USE_STARTER 1

   176 # else

   177 #  if USE___THREAD || (defined USE_TLS && !defined SHARED)

   178     /* These routines are never needed in this configuration.  */

   179 #   define USE_STARTER 0

   180 #  else

   181 #   define USE_STARTER (USE_TLS ? 4 : 1)

   182 #  endif

   183 # endif

   184 #endif

   185 

   186 /*----------------------------------------------------------------------*/

   187 

   188 /* Arenas */

   189 static tsd_key_t arena_key;

   190 static mutex_t list_lock;

   191 

   192 /* Arena structure */

   193 struct malloc_arena {

   194   /* Serialize access.  */

   195   mutex_t mutex;

   196 

   197   /* Statistics for locking.  Only used if THREAD_STATS is defined.  */

   198   long stat_lock_direct, stat_lock_loop, stat_lock_wait;

   199   long stat_starter;

   200 

   201   /* Linked list */

   202   struct malloc_arena *next;

   203 

   204   /* Space for mstate.  The size is just the minimum such that

   205      create_mspace_with_base can be successfully called.  */

   206   char buf_[pad_request(sizeof(struct malloc_state)) + TOP_FOOT_SIZE +

   207 	    CHUNK_ALIGN_MASK + 1];

   208 };

   209 #define MSPACE_OFFSET (((offsetof(struct malloc_arena, buf_) \

   210 			 + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK))

   211 #define arena_to_mspace(a) ((void *)chunk2mem((char*)(a) + MSPACE_OFFSET))

   212 

   213 /* check for chunk from non-main arena */

   214 #define chunk_non_main_arena(p) ((p)->head & NON_MAIN_ARENA)

   215 

   216 static struct malloc_arena* _int_new_arena(size_t size);

   217 

   218 /* Buffer for the main arena. */

   219 static struct malloc_arena main_arena;

   220 

   221 /* For now, store arena in footer.  This means typically 4bytes more

   222    overhead for each non-main-arena chunk, but is fast and easy to

   223    compute.  Note that the pointer stored in the extra footer must be

   224    properly aligned, though. */

   225 #define FOOTER_OVERHEAD \

   226  (2*sizeof(struct malloc_arena*) - SIZE_T_SIZE)

   227 

   228 #define arena_for_chunk(ptr) \

   229  (chunk_non_main_arena(ptr) ? *(struct malloc_arena**)              \

   230   ((char*)(ptr) + chunksize(ptr) - (FOOTER_OVERHEAD - SIZE_T_SIZE)) \

   231   : &main_arena)

   232 

   233 /* special because of extra overhead */

   234 #define arena_for_mmap_chunk(ptr) \

   235  (chunk_non_main_arena(ptr) ? *(struct malloc_arena**)             \

   236   ((char*)(ptr) + chunksize(ptr) - sizeof(struct malloc_arena*))   \

   237   : &main_arena)

   238 

   239 #define set_non_main_arena(mem, ar_ptr) do {                   		      \

   240   mchunkptr P = mem2chunk(mem);                                               \

   241   size_t SZ = chunksize(P) - (is_mmapped(P) ? sizeof(struct malloc_arena*)    \

   242                               : (FOOTER_OVERHEAD - SIZE_T_SIZE));             \

   243   assert((unsigned long)((char*)(P) + SZ)%sizeof(struct malloc_arena*) == 0); \

   244   *(struct malloc_arena**)((char*)(P) + SZ) = (ar_ptr);                       \

   245   P->head |= NON_MAIN_ARENA;                                                  \

   246 } while (0)

   247 

   248 /* arena_get() acquires an arena and locks the corresponding mutex.

   249    First, try the one last locked successfully by this thread.  (This

   250    is the common case and handled with a macro for speed.)  Then, loop

   251    once over the circularly linked list of arenas.  If no arena is

   252    readily available, create a new one.  In this latter case, `size'

   253    is just a hint as to how much memory will be required immediately

   254    in the new arena. */

   255 

   256 #define arena_get(ptr, size) do { \

   257   void *vptr = NULL; \

   258   ptr = (struct malloc_arena*)tsd_getspecific(arena_key, vptr); \

   259   if(ptr && !mutex_trylock(&ptr->mutex)) { \

   260     THREAD_STAT(++(ptr->stat_lock_direct)); \

   261   } else \

   262     ptr = arena_get2(ptr, (size)); \

   263 } while(0)

   264 

   265 static struct malloc_arena*

   266 arena_get2(struct malloc_arena* a_tsd, size_t size)

   267 {

   268   struct malloc_arena* a;

   269   int err;

   270 

   271   if(!a_tsd)

   272     a = a_tsd = &main_arena;

   273   else {

   274     a = a_tsd->next;

   275     if(!a) {

   276       /* This can only happen while initializing the new arena. */

   277       (void)mutex_lock(&main_arena.mutex);

   278       THREAD_STAT(++(main_arena.stat_lock_wait));

   279       return &main_arena;

   280     }

   281   }

   282 

   283   /* Check the global, circularly linked list for available arenas. */

   284  repeat:

   285   do {

   286     if(!mutex_trylock(&a->mutex)) {

   287       THREAD_STAT(++(a->stat_lock_loop));

   288       tsd_setspecific(arena_key, (void *)a);

   289       return a;

   290     }

   291     a = a->next;

   292   } while(a != a_tsd);

   293 

   294   /* If not even the list_lock can be obtained, try again.  This can

   295      happen during `atfork', or for example on systems where thread

   296      creation makes it temporarily impossible to obtain _any_

   297      locks. */

   298   if(mutex_trylock(&list_lock)) {

   299     a = a_tsd;

   300     goto repeat;

   301   }

   302   (void)mutex_unlock(&list_lock);

   303 

   304   /* Nothing immediately available, so generate a new arena.  */

   305   a = _int_new_arena(size);

   306   if(!a)

   307     return 0;

   308 

   309   tsd_setspecific(arena_key, (void *)a);

   310   mutex_init(&a->mutex);

   311   err = mutex_lock(&a->mutex); /* remember result */

   312 

   313   /* Add the new arena to the global list.  */

   314   (void)mutex_lock(&list_lock);

   315   a->next = main_arena.next;

   316   atomic_write_barrier ();

   317   main_arena.next = a;

   318   (void)mutex_unlock(&list_lock);

   319 

   320   if(err) /* locking failed; keep arena for further attempts later */

   321     return 0;

   322 

   323   THREAD_STAT(++(a->stat_lock_loop));

   324   return a;

   325 }

   326 

   327 /* Create a new arena with room for a chunk of size "size".  */

   328 

   329 static struct malloc_arena*

   330 _int_new_arena(size_t size)

   331 {

   332   struct malloc_arena* a;

   333   size_t mmap_sz = sizeof(*a) + pad_request(size);

   334   void *m;

   335 

   336   if (mmap_sz < ARENA_SIZE_MIN)

   337     mmap_sz = ARENA_SIZE_MIN;

   338   /* conservative estimate for page size */

   339   mmap_sz = (mmap_sz + 8191) & ~(size_t)8191;

   340   a = CALL_MMAP(mmap_sz);

   341   if ((char*)a == (char*)-1)

   342     return 0;

   343 

   344   m = create_mspace_with_base((char*)a + MSPACE_OFFSET,

   345 			      mmap_sz - MSPACE_OFFSET,

   346 			      0);

   347 

   348   if (!m) { 

   349     CALL_MUNMAP(a, mmap_sz);

   350     a = 0;

   351   } else {

   352     /*a->next = NULL;*/

   353     /*a->system_mem = a->max_system_mem = h->size;*/

   354   }

   355 

   356   return a;

   357 }

   358 

   359 /*------------------------------------------------------------------------*/

   360 

   361 /* Hook mechanism for proper initialization and atfork support. */

   362 

   363 /* Define and initialize the hook variables.  These weak definitions must

   364    appear before any use of the variables in a function.  */

   365 #ifndef weak_variable

   366 #ifndef _LIBC

   367 #define weak_variable /**/

   368 #else

   369 /* In GNU libc we want the hook variables to be weak definitions to

   370    avoid a problem with Emacs.  */

   371 #define weak_variable weak_function

   372 #endif

   373 #endif

   374 

   375 #if !(USE_STARTER & 2)

   376 # define free_hook_ini     NULL

   377 /* Forward declarations.  */

   378 static void* malloc_hook_ini (size_t sz, const void *caller);

   379 static void* realloc_hook_ini (void* ptr, size_t sz, const void* caller);

   380 static void* memalign_hook_ini (size_t alignment, size_t sz,

   381 				const void* caller);

   382 #else

   383 # define free_hook_ini     free_starter

   384 # define malloc_hook_ini   malloc_starter

   385 # define realloc_hook_ini  NULL

   386 # define memalign_hook_ini memalign_starter

   387 #endif

   388 

   389 void weak_variable (*__malloc_initialize_hook) (void) = NULL;

   390 void weak_variable (*__free_hook) (void * __ptr, const void *)

   391      = free_hook_ini;

   392 void * weak_variable (*__malloc_hook) (size_t __size, const void *)

   393      = malloc_hook_ini;

   394 void * weak_variable (*__realloc_hook)

   395      (void * __ptr, size_t __size, const void *) = realloc_hook_ini;

   396 void * weak_variable (*__memalign_hook)

   397   (size_t __alignment, size_t __size, const void *) = memalign_hook_ini;

   398 /*void weak_variable (*__after_morecore_hook) (void) = NULL;*/

   399 

   400 /* The initial hooks just call the initialization routine, then do the

   401    normal work. */

   402 

   403 #if !(USE_STARTER & 2)

   404 static

   405 #endif

   406 void ptmalloc_init(void);

   407 

   408 #if !(USE_STARTER & 2)

   409 

   410 static void*

   411 malloc_hook_ini(size_t sz, const void * caller)

   412 {

   413   __malloc_hook = NULL;

   414   ptmalloc_init();

   415   return public_mALLOc(sz);

   416 }

   417 

   418 static void *

   419 realloc_hook_ini(void *ptr, size_t sz, const void * caller)

   420 {

   421   __malloc_hook = NULL;

   422   __realloc_hook = NULL;

   423   ptmalloc_init();

   424   return public_rEALLOc(ptr, sz);

   425 }

   426 

   427 static void*

   428 memalign_hook_ini(size_t alignment, size_t sz, const void * caller)

   429 {

   430   __memalign_hook = NULL;

   431   ptmalloc_init();

   432   return public_mEMALIGn(alignment, sz);

   433 }

   434 

   435 #endif /* !(USE_STARTER & 2) */

   436 

   437 /*----------------------------------------------------------------------*/

   438 

   439 #if !defined NO_THREADS && USE_STARTER

   440 

   441 /* The following hooks are used when the global initialization in

   442    ptmalloc_init() hasn't completed yet. */

   443 

   444 static void*

   445 malloc_starter(size_t sz, const void *caller)

   446 {

   447   void* victim;

   448 

   449   /*ptmalloc_init_minimal();*/

   450   victim = mspace_malloc(arena_to_mspace(&main_arena), sz);

   451   THREAD_STAT(++main_arena.stat_starter);

   452 

   453   return victim;

   454 }

   455 

   456 static void*

   457 memalign_starter(size_t align, size_t sz, const void *caller)

   458 {

   459   void* victim;

   460 

   461   /*ptmalloc_init_minimal();*/

   462   victim = mspace_memalign(arena_to_mspace(&main_arena), align, sz);

   463   THREAD_STAT(++main_arena.stat_starter);

   464 

   465   return victim;

   466 }

   467 

   468 static void

   469 free_starter(void* mem, const void *caller)

   470 {

   471   if (mem) {

   472     mchunkptr p = mem2chunk(mem);

   473     void *msp = arena_to_mspace(&main_arena);

   474     if (is_mmapped(p))

   475       munmap_chunk(msp, p);

   476     else

   477       mspace_free(msp, mem);

   478   }

   479   THREAD_STAT(++main_arena.stat_starter);

   480 }

   481 

   482 #endif /* !defined NO_THREADS && USE_STARTER */

   483 

   484 /*----------------------------------------------------------------------*/

   485 

   486 #ifndef NO_THREADS

   487 

   488 /* atfork support.  */

   489 

   490 static void * (*save_malloc_hook) (size_t __size, const void *);

   491 # if !defined _LIBC || !defined USE_TLS || (defined SHARED && !USE___THREAD)

   492 static void * (*save_memalign_hook) (size_t __align, size_t __size,

   493 				     const void *);

   494 # endif

   495 static void   (*save_free_hook) (void * __ptr, const void *);

   496 static void*  save_arena;

   497 

   498 /* Magic value for the thread-specific arena pointer when

   499    malloc_atfork() is in use.  */

   500 

   501 #define ATFORK_ARENA_PTR ((void*)-1)

   502 

   503 /* The following hooks are used while the `atfork' handling mechanism

   504    is active. */

   505 

   506 static void*

   507 malloc_atfork(size_t sz, const void *caller)

   508 {

   509   void *vptr = NULL;

   510 

   511   tsd_getspecific(arena_key, vptr);

   512   if(vptr == ATFORK_ARENA_PTR) {

   513     /* We are the only thread that may allocate at all.  */

   514     return mspace_malloc(arena_to_mspace(&main_arena), sz);

   515   } else {

   516     /* Suspend the thread until the `atfork' handlers have completed.

   517        By that time, the hooks will have been reset as well, so that

   518        mALLOc() can be used again. */

   519     (void)mutex_lock(&list_lock);

   520     (void)mutex_unlock(&list_lock);

   521     return public_mALLOc(sz);

   522   }

   523 }

   524 

   525 static void

   526 free_atfork(void* mem, const void *caller)

   527 {

   528   void *vptr = NULL;

   529   struct malloc_arena *ar_ptr;

   530   mchunkptr p;                          /* chunk corresponding to mem */

   531 

   532   if (mem == 0)                              /* free(0) has no effect */

   533     return;

   534 

   535   p = mem2chunk(mem);

   536 

   537   if (is_mmapped(p)) {                      /* release mmapped memory. */

   538     ar_ptr = arena_for_mmap_chunk(p);

   539     munmap_chunk(arena_to_mspace(ar_ptr), p);

   540     return;

   541   }

   542 

   543   ar_ptr = arena_for_chunk(p);

   544   tsd_getspecific(arena_key, vptr);

   545   if(vptr != ATFORK_ARENA_PTR)

   546     (void)mutex_lock(&ar_ptr->mutex);

   547   mspace_free(arena_to_mspace(ar_ptr), mem);

   548   if(vptr != ATFORK_ARENA_PTR)

   549     (void)mutex_unlock(&ar_ptr->mutex);

   550 }

   551 

   552 /* The following two functions are registered via thread_atfork() to

   553    make sure that the mutexes remain in a consistent state in the

   554    fork()ed version of a thread.  Also adapt the malloc and free hooks

   555    temporarily, because the `atfork' handler mechanism may use

   556    malloc/free internally (e.g. in LinuxThreads). */

   557 

   558 static void

   559 ptmalloc_lock_all (void)

   560 {

   561   struct malloc_arena* ar_ptr;

   562 

   563   if(__malloc_initialized < 1)

   564     return;

   565   (void)mutex_lock(&list_lock);

   566   for(ar_ptr = &main_arena;;) {

   567     (void)mutex_lock(&ar_ptr->mutex);

   568     ar_ptr = ar_ptr->next;

   569     if(ar_ptr == &main_arena)

   570       break;

   571   }

   572   save_malloc_hook = __malloc_hook;

   573   save_free_hook = __free_hook;

   574   __malloc_hook = malloc_atfork;

   575   __free_hook = free_atfork;

   576   /* Only the current thread may perform malloc/free calls now. */

   577   tsd_getspecific(arena_key, save_arena);

   578   tsd_setspecific(arena_key, ATFORK_ARENA_PTR);

   579 }

   580 

   581 static void

   582 ptmalloc_unlock_all (void)

   583 {

   584   struct malloc_arena *ar_ptr;

   585 

   586   if(__malloc_initialized < 1)

   587     return;

   588   tsd_setspecific(arena_key, save_arena);

   589   __malloc_hook = save_malloc_hook;

   590   __free_hook = save_free_hook;

   591   for(ar_ptr = &main_arena;;) {

   592     (void)mutex_unlock(&ar_ptr->mutex);

   593     ar_ptr = ar_ptr->next;

   594     if(ar_ptr == &main_arena) break;

   595   }

   596   (void)mutex_unlock(&list_lock);

   597 }

   598 

   599 #ifdef __linux__

   600 

   601 /* In LinuxThreads, unlocking a mutex in the child process after a

   602    fork() is currently unsafe, whereas re-initializing it is safe and

   603    does not leak resources.  Therefore, a special atfork handler is

   604    installed for the child. */

   605 

   606 static void

   607 ptmalloc_unlock_all2(void)

   608 {

   609   struct malloc_arena *ar_ptr;

   610 

   611   if(__malloc_initialized < 1)

   612     return;

   613 #if defined _LIBC || 1 /*defined MALLOC_HOOKS*/

   614   tsd_setspecific(arena_key, save_arena);

   615   __malloc_hook = save_malloc_hook;

   616   __free_hook = save_free_hook;

   617 #endif

   618   for(ar_ptr = &main_arena;;) {

   619     (void)mutex_init(&ar_ptr->mutex);

   620     ar_ptr = ar_ptr->next;

   621     if(ar_ptr == &main_arena) break;

   622   }

   623   (void)mutex_init(&list_lock);

   624 }

   625 

   626 #else

   627 

   628 #define ptmalloc_unlock_all2 ptmalloc_unlock_all

   629 

   630 #endif

   631 

   632 #endif /* !defined NO_THREADS */

   633 

   634 /*---------------------------------------------------------------------*/

   635 

   636 #if !(USE_STARTER & 2)

   637 static

   638 #endif

   639 void

   640 ptmalloc_init(void)

   641 {

   642   const char* s;

   643   int secure = 0;

   644   void *mspace;

   645 

   646   if(__malloc_initialized >= 0) return;

   647   __malloc_initialized = 0;

   648 

   649   /*if (mp_.pagesize == 0)

   650     ptmalloc_init_minimal();*/

   651 

   652 #ifndef NO_THREADS

   653 # if USE_STARTER & 1

   654   /* With some threads implementations, creating thread-specific data

   655      or initializing a mutex may call malloc() itself.  Provide a

   656      simple starter version (realloc() won't work). */

   657   save_malloc_hook = __malloc_hook;

   658   save_memalign_hook = __memalign_hook;

   659   save_free_hook = __free_hook;

   660   __malloc_hook = malloc_starter;

   661   __memalign_hook = memalign_starter;

   662   __free_hook = free_starter;

   663 #  ifdef _LIBC

   664   /* Initialize the pthreads interface. */

   665   if (__pthread_initialize != NULL)

   666     __pthread_initialize();

   667 #  endif /* !defined _LIBC */

   668 # endif /* USE_STARTER & 1 */

   669 #endif /* !defined NO_THREADS */

   670   mutex_init(&main_arena.mutex);

   671   main_arena.next = &main_arena;

   672   mspace = create_mspace_with_base((char*)&main_arena + MSPACE_OFFSET,

   673 				   sizeof(main_arena) - MSPACE_OFFSET,

   674 				   0);

   675   assert(mspace == arena_to_mspace(&main_arena));

   676 

   677   mutex_init(&list_lock);

   678   tsd_key_create(&arena_key, NULL);

   679   tsd_setspecific(arena_key, (void *)&main_arena);

   680   thread_atfork(ptmalloc_lock_all, ptmalloc_unlock_all, ptmalloc_unlock_all2);

   681 #ifndef NO_THREADS

   682 # if USE_STARTER & 1

   683   __malloc_hook = save_malloc_hook;

   684   __memalign_hook = save_memalign_hook;

   685   __free_hook = save_free_hook;

   686 # endif

   687 # if USE_STARTER & 2

   688   __malloc_hook = 0;

   689   __memalign_hook = 0;

   690   __free_hook = 0;

   691 # endif

   692 #endif

   693 #ifdef _LIBC

   694   secure = __libc_enable_secure;

   695 #else

   696   if (! secure) {

   697     if ((s = getenv("MALLOC_TRIM_THRESHOLD_")))

   698       public_mALLOPt(M_TRIM_THRESHOLD, atoi(s));

   699     if ((s = getenv("MALLOC_TOP_PAD_")) ||

   700 	(s = getenv("MALLOC_GRANULARITY_")))

   701       public_mALLOPt(M_GRANULARITY, atoi(s));

   702     if ((s = getenv("MALLOC_MMAP_THRESHOLD_")))

   703       public_mALLOPt(M_MMAP_THRESHOLD, atoi(s));

   704     /*if ((s = getenv("MALLOC_MMAP_MAX_"))) this is no longer available

   705       public_mALLOPt(M_MMAP_MAX, atoi(s));*/

   706   }

   707   s = getenv("MALLOC_CHECK_");

   708 #endif

   709   if (s) {

   710     /*if(s[0]) mALLOPt(M_CHECK_ACTION, (int)(s[0] - '0'));

   711       __malloc_check_init();*/

   712   }

   713   if (__malloc_initialize_hook != NULL)

   714     (*__malloc_initialize_hook)();

   715   __malloc_initialized = 1;

   716 }

   717 

   718 /*------------------------ Public wrappers. --------------------------------*/

   719 

   720 void*

   721 public_mALLOc(size_t bytes)

   722 {

   723   struct malloc_arena* ar_ptr;

   724   void *victim;

   725 

   726   void * (*hook) (size_t, const void *) = __malloc_hook;

   727   if (hook != NULL)

   728     return (*hook)(bytes, RETURN_ADDRESS (0));

   729 

   730   arena_get(ar_ptr, bytes + FOOTER_OVERHEAD);

   731   if (!ar_ptr)

   732     return 0;

   733   if (ar_ptr != &main_arena)

   734     bytes += FOOTER_OVERHEAD;

   735   victim = mspace_malloc(arena_to_mspace(ar_ptr), bytes);

   736   if (victim && ar_ptr != &main_arena)

   737     set_non_main_arena(victim, ar_ptr);

   738   (void)mutex_unlock(&ar_ptr->mutex);

   739   assert(!victim || is_mmapped(mem2chunk(victim)) ||

   740 	 ar_ptr == arena_for_chunk(mem2chunk(victim)));

   741   return victim;

   742 }

   743 #ifdef libc_hidden_def

   744 libc_hidden_def(public_mALLOc)

   745 #endif

   746 

   747 void

   748 public_fREe(void* mem)

   749 {

   750   struct malloc_arena* ar_ptr;

   751   mchunkptr p;                          /* chunk corresponding to mem */

   752 

   753   void (*hook) (void *, const void *) = __free_hook;

   754   if (hook != NULL) {

   755     (*hook)(mem, RETURN_ADDRESS (0));

   756     return;

   757   }

   758 

   759   if (mem == 0)                              /* free(0) has no effect */

   760     return;

   761 

   762   p = mem2chunk(mem);

   763 

   764   if (is_mmapped(p)) {                      /* release mmapped memory. */

   765     ar_ptr = arena_for_mmap_chunk(p);

   766     munmap_chunk(arena_to_mspace(ar_ptr), p);

   767     return;

   768   }

   769 

   770   ar_ptr = arena_for_chunk(p);

   771 #if THREAD_STATS

   772   if(!mutex_trylock(&ar_ptr->mutex))

   773     ++(ar_ptr->stat_lock_direct);

   774   else {

   775     (void)mutex_lock(&ar_ptr->mutex);

   776     ++(ar_ptr->stat_lock_wait);

   777   }

   778 #else

   779   (void)mutex_lock(&ar_ptr->mutex);

   780 #endif

   781   mspace_free(arena_to_mspace(ar_ptr), mem);

   782   (void)mutex_unlock(&ar_ptr->mutex);

   783 }

   784 #ifdef libc_hidden_def

   785 libc_hidden_def (public_fREe)

   786 #endif

   787 

   788 void*

   789 public_rEALLOc(void* oldmem, size_t bytes)

   790 {

   791   struct malloc_arena* ar_ptr;

   792 

   793   mchunkptr oldp;             /* chunk corresponding to oldmem */

   794 

   795   void* newp;             /* chunk to return */

   796 

   797   void * (*hook) (void *, size_t, const void *) = __realloc_hook;

   798   if (hook != NULL)

   799     return (*hook)(oldmem, bytes, RETURN_ADDRESS (0));

   800 

   801 #if REALLOC_ZERO_BYTES_FREES

   802   if (bytes == 0 && oldmem != NULL) { public_fREe(oldmem); return 0; }

   803 #endif

   804 

   805   /* realloc of null is supposed to be same as malloc */

   806   if (oldmem == 0)

   807     return public_mALLOc(bytes);

   808 

   809   oldp    = mem2chunk(oldmem);

   810   if (is_mmapped(oldp))

   811     ar_ptr = arena_for_mmap_chunk(oldp); /* FIXME: use mmap_resize */

   812   else

   813     ar_ptr = arena_for_chunk(oldp);

   814 #if THREAD_STATS

   815   if(!mutex_trylock(&ar_ptr->mutex))

   816     ++(ar_ptr->stat_lock_direct);

   817   else {

   818     (void)mutex_lock(&ar_ptr->mutex);

   819     ++(ar_ptr->stat_lock_wait);

   820   }

   821 #else

   822   (void)mutex_lock(&ar_ptr->mutex);

   823 #endif

   824 

   825 #ifndef NO_THREADS

   826   /* As in malloc(), remember this arena for the next allocation. */

   827   tsd_setspecific(arena_key, (void *)ar_ptr);

   828 #endif

   829 

   830   if (ar_ptr != &main_arena)

   831     bytes += FOOTER_OVERHEAD;

   832   newp = mspace_realloc(arena_to_mspace(ar_ptr), oldmem, bytes);

   833 

   834   if (newp && ar_ptr != &main_arena)

   835     set_non_main_arena(newp, ar_ptr);

   836   (void)mutex_unlock(&ar_ptr->mutex);

   837 

   838   assert(!newp || is_mmapped(mem2chunk(newp)) ||

   839 	 ar_ptr == arena_for_chunk(mem2chunk(newp)));

   840   return newp;

   841 }

   842 #ifdef libc_hidden_def

   843 libc_hidden_def (public_rEALLOc)

   844 #endif

   845 

   846 void*

   847 public_mEMALIGn(size_t alignment, size_t bytes)

   848 {

   849   struct malloc_arena* ar_ptr;

   850   void *p;

   851 

   852   void * (*hook) (size_t, size_t, const void *) = __memalign_hook;

   853   if (hook != NULL)

   854     return (*hook)(alignment, bytes, RETURN_ADDRESS (0));

   855 

   856   /* If need less alignment than we give anyway, just relay to malloc */

   857   if (alignment <= MALLOC_ALIGNMENT) return public_mALLOc(bytes);

   858 

   859   /* Otherwise, ensure that it is at least a minimum chunk size */

   860   if (alignment <  MIN_CHUNK_SIZE)

   861     alignment = MIN_CHUNK_SIZE;

   862 

   863   arena_get(ar_ptr,

   864 	    bytes + FOOTER_OVERHEAD + alignment + MIN_CHUNK_SIZE);

   865   if(!ar_ptr)

   866     return 0;

   867 

   868   if (ar_ptr != &main_arena)

   869     bytes += FOOTER_OVERHEAD;

   870   p = mspace_memalign(arena_to_mspace(ar_ptr), alignment, bytes);

   871 

   872   if (p && ar_ptr != &main_arena)

   873     set_non_main_arena(p, ar_ptr);

   874   (void)mutex_unlock(&ar_ptr->mutex);

   875 

   876   assert(!p || is_mmapped(mem2chunk(p)) ||

   877 	 ar_ptr == arena_for_chunk(mem2chunk(p)));

   878   return p;

   879 }

   880 #ifdef libc_hidden_def

   881 libc_hidden_def (public_mEMALIGn)

   882 #endif

   883 

   884 void*

   885 public_vALLOc(size_t bytes)

   886 {

   887   struct malloc_arena* ar_ptr;

   888   void *p;

   889 

   890   if(__malloc_initialized < 0)

   891     ptmalloc_init ();

   892   arena_get(ar_ptr, bytes + FOOTER_OVERHEAD + MIN_CHUNK_SIZE);

   893   if(!ar_ptr)

   894     return 0;

   895   if (ar_ptr != &main_arena)

   896     bytes += FOOTER_OVERHEAD;

   897   p = mspace_memalign(arena_to_mspace(ar_ptr), 4096, bytes);

   898 

   899   if (p && ar_ptr != &main_arena)

   900     set_non_main_arena(p, ar_ptr);

   901   (void)mutex_unlock(&ar_ptr->mutex);

   902   return p;

   903 }

   904 

   905 int

   906 public_pMEMALIGn (void **memptr, size_t alignment, size_t size)

   907 {

   908   void *mem;

   909 

   910   /* Test whether the SIZE argument is valid.  It must be a power of

   911      two multiple of sizeof (void *).  */

   912   if (alignment % sizeof (void *) != 0

   913       || !my_powerof2 (alignment / sizeof (void *)) != 0

   914       || alignment == 0)

   915     return EINVAL;

   916 

   917   mem = public_mEMALIGn (alignment, size);

   918 

   919   if (mem != NULL) {

   920     *memptr = mem;

   921     return 0;

   922   }

   923 

   924   return ENOMEM;

   925 }

   926 

   927 void*

   928 public_cALLOc(size_t n_elements, size_t elem_size)

   929 {

   930   struct malloc_arena* ar_ptr;

   931   size_t bytes, sz;

   932   void* mem;

   933   void * (*hook) (size_t, const void *) = __malloc_hook;

   934 

   935   /* size_t is unsigned so the behavior on overflow is defined.  */

   936   bytes = n_elements * elem_size;

   937 #define HALF_INTERNAL_SIZE_T \

   938   (((size_t) 1) << (8 * sizeof (size_t) / 2))

   939   if (__builtin_expect ((n_elements | elem_size) >= HALF_INTERNAL_SIZE_T, 0)) {

   940     if (elem_size != 0 && bytes / elem_size != n_elements) {

   941       /*MALLOC_FAILURE_ACTION;*/

   942       return 0;

   943     }

   944   }

   945 

   946   if (hook != NULL) {

   947     sz = bytes;

   948     mem = (*hook)(sz, RETURN_ADDRESS (0));

   949     if(mem == 0)

   950       return 0;

   951 #ifdef HAVE_MEMCPY

   952     return memset(mem, 0, sz);

   953 #else

   954     while(sz > 0) ((char*)mem)[--sz] = 0; /* rather inefficient */

   955     return mem;

   956 #endif

   957   }

   958 

   959   arena_get(ar_ptr, bytes + FOOTER_OVERHEAD);

   960   if(!ar_ptr)

   961     return 0;

   962 

   963   if (ar_ptr != &main_arena)

   964     bytes += FOOTER_OVERHEAD;

   965   mem = mspace_calloc(arena_to_mspace(ar_ptr), bytes, 1);

   966 

   967   if (mem && ar_ptr != &main_arena)

   968     set_non_main_arena(mem, ar_ptr);

   969   (void)mutex_unlock(&ar_ptr->mutex);

   970   

   971   assert(!mem || is_mmapped(mem2chunk(mem)) ||

   972 	 ar_ptr == arena_for_chunk(mem2chunk(mem)));

   973 

   974   return mem;

   975 }

   976 

   977 void**

   978 public_iCALLOc(size_t n, size_t elem_size, void* chunks[])

   979 {

   980   struct malloc_arena* ar_ptr;

   981   void** m;

   982 

   983   arena_get(ar_ptr, n*(elem_size + FOOTER_OVERHEAD));

   984   if (!ar_ptr)

   985     return 0;

   986 

   987   if (ar_ptr != &main_arena)

   988     elem_size += FOOTER_OVERHEAD;

   989   m = mspace_independent_calloc(arena_to_mspace(ar_ptr), n, elem_size, chunks);

   990 

   991   if (m && ar_ptr != &main_arena) {

   992     while (n > 0)

   993       set_non_main_arena(m[--n], ar_ptr);

   994   }

   995   (void)mutex_unlock(&ar_ptr->mutex);

   996   return m;

   997 }

   998 

   999 void**

  1000 public_iCOMALLOc(size_t n, size_t sizes[], void* chunks[])

  1001 {

  1002   struct malloc_arena* ar_ptr;

  1003   size_t* m_sizes;

  1004   size_t i;

  1005   void** m;

  1006 

  1007   arena_get(ar_ptr, n*sizeof(size_t));

  1008   if (!ar_ptr)

  1009     return 0;

  1010 

  1011   if (ar_ptr != &main_arena) {

  1012     /* Temporary m_sizes[] array is ugly but it would be surprising to

  1013        change the original sizes[]... */

  1014     m_sizes = mspace_malloc(arena_to_mspace(ar_ptr), n*sizeof(size_t));

  1015     if (!m_sizes) {

  1016       (void)mutex_unlock(&ar_ptr->mutex);

  1017       return 0;

  1018     }

  1019     for (i=0; i<n; ++i)

  1020       m_sizes[i] = sizes[i] + FOOTER_OVERHEAD;

  1021     if (!chunks) {

  1022       chunks = mspace_malloc(arena_to_mspace(ar_ptr),

  1023 			     n*sizeof(void*)+FOOTER_OVERHEAD);

  1024       if (!chunks) {

  1025 	mspace_free(arena_to_mspace(ar_ptr), m_sizes);

  1026 	(void)mutex_unlock(&ar_ptr->mutex);

  1027 	return 0;

  1028       }

  1029       set_non_main_arena(chunks, ar_ptr);

  1030     }

  1031   } else

  1032     m_sizes = sizes;

  1033 

  1034   m = mspace_independent_comalloc(arena_to_mspace(ar_ptr), n, m_sizes, chunks);

  1035 

  1036   if (ar_ptr != &main_arena) {

  1037     mspace_free(arena_to_mspace(ar_ptr), m_sizes);

  1038     if (m)

  1039       for (i=0; i<n; ++i)

  1040 	set_non_main_arena(m[i], ar_ptr);

  1041   }

  1042   (void)mutex_unlock(&ar_ptr->mutex);

  1043   return m;

  1044 }

  1045 

  1046 #if 0 && !defined _LIBC

  1047 

  1048 void

  1049 public_cFREe(void* m)

  1050 {

  1051   public_fREe(m);

  1052 }

  1053 

  1054 #endif /* _LIBC */

  1055 

  1056 int

  1057 public_mTRIm(size_t s)

  1058 {

  1059   int result;

  1060 

  1061   (void)mutex_lock(&main_arena.mutex);

  1062   result = mspace_trim(arena_to_mspace(&main_arena), s);

  1063   (void)mutex_unlock(&main_arena.mutex);

  1064   return result;

  1065 }

  1066 

  1067 size_t

  1068 public_mUSABLe(void* mem)

  1069 {

  1070   if (mem != 0) {

  1071     mchunkptr p = mem2chunk(mem);

  1072     if (cinuse(p))

  1073       return chunksize(p) - overhead_for(p);

  1074   }

  1075   return 0;

  1076 }

  1077 

  1078 int

  1079 public_mALLOPt(int p, int v)

  1080 {

  1081   int result;

  1082   result = mspace_mallopt(p, v);

  1083   return result;

  1084 }

  1085 

  1086 void

  1087 public_mSTATs(void)

  1088 {

  1089   int i;

  1090   struct malloc_arena* ar_ptr;

  1091   /*unsigned long in_use_b, system_b, avail_b;*/

  1092 #if THREAD_STATS

  1093   long stat_lock_direct = 0, stat_lock_loop = 0, stat_lock_wait = 0;

  1094 #endif

  1095 

  1096   if(__malloc_initialized < 0)

  1097     ptmalloc_init ();

  1098   for (i=0, ar_ptr = &main_arena;; ++i) {

  1099     struct malloc_state* msp = arena_to_mspace(ar_ptr);

  1100 

  1101     fprintf(stderr, "Arena %d:\n", i);

  1102     mspace_malloc_stats(msp);

  1103 #if THREAD_STATS

  1104     stat_lock_direct += ar_ptr->stat_lock_direct;

  1105     stat_lock_loop += ar_ptr->stat_lock_loop;

  1106     stat_lock_wait += ar_ptr->stat_lock_wait;

  1107 #endif

  1108     if (MALLOC_DEBUG > 1) {

  1109       struct malloc_segment* mseg = &msp->seg;

  1110       while (mseg) {

  1111 	fprintf(stderr, " seg %08lx-%08lx\n", (unsigned long)mseg->base,

  1112 		(unsigned long)(mseg->base + mseg->size));

  1113 	mseg = mseg->next;

  1114       }

  1115     }

  1116     ar_ptr = ar_ptr->next;

  1117     if (ar_ptr == &main_arena)

  1118       break;

  1119   }

  1120 #if THREAD_STATS

  1121   fprintf(stderr, "locked directly  = %10ld\n", stat_lock_direct);

  1122   fprintf(stderr, "locked in loop   = %10ld\n", stat_lock_loop);

  1123   fprintf(stderr, "locked waiting   = %10ld\n", stat_lock_wait);

  1124   fprintf(stderr, "locked total     = %10ld\n",

  1125           stat_lock_direct + stat_lock_loop + stat_lock_wait);

  1126   if (main_arena.stat_starter > 0)

  1127     fprintf(stderr, "starter hooks    = %10ld\n", main_arena.stat_starter);

  1128 #endif

  1129 }

  1130 

  1131 /*

  1132  * Local variables:

  1133  * c-basic-offset: 2

  1134  * End:

  1135  */