1 # 2005 November 30

     2 #

     3 # The author disclaims copyright to this source code.  In place of

     4 # a legal notice, here is a blessing:

     5 #

     6 #    May you do good and not evil.

     7 #    May you find forgiveness for yourself and forgive others.

     8 #    May you share freely, never taking more than you give.

     9 #

    10 #***********************************************************************

    11 #

    12 # This file contains test cases focused on the two memory-management APIs, 

    13 # sqlite3_soft_heap_limit() and sqlite3_release_memory().

    14 #

    15 # Prior to version 3.6.2, calling sqlite3_release_memory() or exceeding

    16 # the configured soft heap limit could cause sqlite to upgrade database 

    17 # locks and flush dirty pages to the file system. As of 3.6.2, this is

    18 # no longer the case. In version 3.6.2, sqlite3_release_memory() only

    19 # reclaims clean pages. This test file has been updated accordingly.

    20 #

    21 # $Id: malloc5.test,v 1.20 2008/08/27 16:38:57 danielk1977 Exp $

    22 

    23 set testdir [file dirname $argv0]

    24 source $testdir/tester.tcl

    25 source $testdir/malloc_common.tcl

    26 db close

    27 

    28 # Only run these tests if memory debugging is turned on.

    29 #

    30 if {!$MEMDEBUG} {

    31    puts "Skipping malloc5 tests: not compiled with -DSQLITE_MEMDEBUG..."

    32    finish_test

    33    return

    34 }

    35 

    36 # Skip these tests if OMIT_MEMORY_MANAGEMENT was defined at compile time.

    37 ifcapable !memorymanage {

    38    finish_test

    39    return

    40 }

    41 

    42 sqlite3_soft_heap_limit 0

    43 sqlite3 db test.db

    44 

    45 do_test malloc5-1.1 {

    46   # Simplest possible test. Call sqlite3_release_memory when there is exactly

    47   # one unused page in a single pager cache. The page cannot be freed, as

    48   # it is dirty. So sqlite3_release_memory() returns 0.

    49   #

    50   execsql {

    51     PRAGMA auto_vacuum=OFF;

    52     BEGIN;

    53     CREATE TABLE abc(a, b, c);

    54   }

    55   sqlite3_release_memory

    56 } {0}

    57 

    58 do_test malloc5-1.2 {

    59   # Test that the transaction started in the above test is still active.

    60   # The lock on the database file should not have been upgraded (this was

    61   # not the case before version 3.6.2).

    62   #

    63   sqlite3 db2 test.db

    64   execsql { SELECT * FROM sqlite_master } db2

    65 } {}

    66 do_test malloc5-1.3 {

    67   # Call [sqlite3_release_memory] when there is exactly one unused page 

    68   # in the cache belonging to db2.

    69   #

    70   set ::pgalloc [sqlite3_release_memory]

    71   expr $::pgalloc > 0

    72 } {1}

    73 

    74 do_test malloc5-1.4 {

    75   # Commit the transaction and open a new one. Read 1 page into the cache.

    76   # Because the page is not dirty, it is eligible for collection even

    77   # before the transaction is concluded.

    78   #

    79   execsql {

    80     COMMIT;

    81     BEGIN;

    82     SELECT * FROM abc;

    83   }

    84   sqlite3_release_memory

    85 } $::pgalloc

    86 

    87 do_test malloc5-1.5 {

    88   # Conclude the transaction opened in the previous [do_test] block. This

    89   # causes another page (page 1) to become eligible for recycling.

    90   #

    91   execsql { COMMIT }

    92   sqlite3_release_memory

    93 } $::pgalloc

    94 

    95 do_test malloc5-1.6 {

    96   # Manipulate the cache so that it contains two unused pages. One requires 

    97   # a journal-sync to free, the other does not.

    98   db2 close

    99   execsql {

   100     BEGIN;

   101     SELECT * FROM abc;

   102     CREATE TABLE def(d, e, f);

   103   }

   104   sqlite3_release_memory 500

   105 } $::pgalloc

   106 

   107 do_test malloc5-1.7 {

   108   # Database should not be locked this time. 

   109   sqlite3 db2 test.db

   110   catchsql { SELECT * FROM abc } db2

   111 } {0 {}}

   112 do_test malloc5-1.8 {

   113   # Try to release another block of memory. This will fail as the only

   114   # pages currently in the cache are dirty (page 3) or pinned (page 1).

   115   db2 close

   116   sqlite3_release_memory 500

   117 } 0

   118 do_test malloc5-1.8 {

   119   # Database is still not locked.

   120   #

   121   sqlite3 db2 test.db

   122   catchsql { SELECT * FROM abc } db2

   123 } {0 {}}

   124 do_test malloc5-1.9 {

   125   execsql {

   126     COMMIT;

   127   }

   128 } {}

   129 

   130 do_test malloc5-2.1 {

   131   # Put some data in tables abc and def. Both tables are still wholly 

   132   # contained within their root pages.

   133   execsql {

   134     INSERT INTO abc VALUES(1, 2, 3);

   135     INSERT INTO abc VALUES(4, 5, 6);

   136     INSERT INTO def VALUES(7, 8, 9);

   137     INSERT INTO def VALUES(10,11,12);

   138   }

   139 } {}

   140 do_test malloc5-2.2 {

   141   # Load the root-page for table def into the cache. Then query table abc. 

   142   # Halfway through the query call sqlite3_release_memory(). The goal of this

   143   # test is to make sure we don't free pages that are in use (specifically, 

   144   # the root of table abc).

   145   sqlite3_release_memory

   146   set nRelease 0

   147   execsql { 

   148     BEGIN;

   149     SELECT * FROM def;

   150   }

   151   set data [list]

   152   db eval {SELECT * FROM abc} {

   153     incr nRelease [sqlite3_release_memory]

   154     lappend data $a $b $c

   155   }

   156   execsql {

   157     COMMIT;

   158   }

   159   list $nRelease $data

   160 } [list $pgalloc [list 1 2 3 4 5 6]]

   161 

   162 do_test malloc5-3.1 {

   163   # Simple test to show that if two pagers are opened from within this

   164   # thread, memory is freed from both when sqlite3_release_memory() is

   165   # called.

   166   execsql {

   167     BEGIN;

   168     SELECT * FROM abc;

   169   }

   170   execsql {

   171     SELECT * FROM sqlite_master;

   172     BEGIN;

   173     SELECT * FROM def;

   174   } db2

   175   sqlite3_release_memory

   176 } [expr $::pgalloc * 2]

   177 do_test malloc5-3.2 {

   178   concat \

   179     [execsql {SELECT * FROM abc; COMMIT}] \

   180     [execsql {SELECT * FROM def; COMMIT} db2]

   181 } {1 2 3 4 5 6 7 8 9 10 11 12}

   182 

   183 db2 close

   184 puts "Highwater mark: [sqlite3_memory_highwater]"

   185 

   186 # The following two test cases each execute a transaction in which 

   187 # 10000 rows are inserted into table abc. The first test case is used

   188 # to ensure that more than 1MB of dynamic memory is used to perform

   189 # the transaction. 

   190 #

   191 # The second test case sets the "soft-heap-limit" to 100,000 bytes (0.1 MB)

   192 # and tests to see that this limit is not exceeded at any point during 

   193 # transaction execution.

   194 #

   195 # Before executing malloc5-4.* we save the value of the current soft heap 

   196 # limit in variable ::soft_limit. The original value is restored after 

   197 # running the tests.

   198 #

   199 set ::soft_limit [sqlite3_soft_heap_limit -1]

   200 execsql {PRAGMA cache_size=2000}

   201 do_test malloc5-4.1 {

   202   execsql {BEGIN;}

   203   execsql {DELETE FROM abc;}

   204   for {set i 0} {$i < 10000} {incr i} {

   205     execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"

   206   }

   207   execsql {COMMIT;}

   208   sqlite3_release_memory

   209   sqlite3_memory_highwater 1

   210   execsql {SELECT * FROM abc}

   211   set nMaxBytes [sqlite3_memory_highwater 1]

   212   puts -nonewline " (Highwater mark: $nMaxBytes) "

   213   expr $nMaxBytes > 1000000

   214 } {1}

   215 do_test malloc5-4.2 {

   216   sqlite3_release_memory

   217   sqlite3_soft_heap_limit 100000

   218   sqlite3_memory_highwater 1

   219   execsql {SELECT * FROM abc}

   220   set nMaxBytes [sqlite3_memory_highwater 1]

   221   puts -nonewline " (Highwater mark: $nMaxBytes) "

   222   expr $nMaxBytes <= 100000

   223 } {1}

   224 do_test malloc5-4.3 {

   225   # Check that the content of table abc is at least roughly as expected.

   226   execsql {

   227     SELECT count(*), sum(a), sum(b) FROM abc;

   228   }

   229 } [list 10000 [expr int(10000.0 * 4999.5)] [expr int(10000.0 * 4999.5)]]

   230 

   231 # Restore the soft heap limit.

   232 sqlite3_soft_heap_limit $::soft_limit

   233 

   234 # Test that there are no problems calling sqlite3_release_memory when

   235 # there are open in-memory databases.

   236 #

   237 # At one point these tests would cause a seg-fault.

   238 #

   239 do_test malloc5-5.1 {

   240   db close

   241   sqlite3 db :memory:

   242   execsql {

   243     BEGIN;

   244     CREATE TABLE abc(a, b, c);

   245     INSERT INTO abc VALUES('abcdefghi', 1234567890, NULL);

   246     INSERT INTO abc SELECT * FROM abc;

   247     INSERT INTO abc SELECT * FROM abc;

   248     INSERT INTO abc SELECT * FROM abc;

   249     INSERT INTO abc SELECT * FROM abc;

   250     INSERT INTO abc SELECT * FROM abc;

   251     INSERT INTO abc SELECT * FROM abc;

   252     INSERT INTO abc SELECT * FROM abc;

   253   }

   254   sqlite3_release_memory

   255 } 0

   256 do_test malloc5-5.2 {

   257   sqlite3_soft_heap_limit 5000

   258   execsql {

   259     COMMIT;

   260     PRAGMA temp_store = memory;

   261     SELECT * FROM abc ORDER BY a;

   262   }

   263   expr 1

   264 } {1}

   265 sqlite3_soft_heap_limit $::soft_limit

   266 

   267 #-------------------------------------------------------------------------

   268 # The following test cases (malloc5-6.*) test the new global LRU list

   269 # used to determine the pages to recycle when sqlite3_release_memory is

   270 # called and there is more than one pager open.

   271 #

   272 proc nPage {db} {

   273   set bt [btree_from_db $db]

   274   array set stats [btree_pager_stats $bt]

   275   set stats(page)

   276 }

   277 db close

   278 file delete -force test.db test.db-journal test2.db test2.db-journal

   279 

   280 # This block of test-cases (malloc5-6.1.*) prepares two database files

   281 # for the subsequent tests.

   282 do_test malloc5-6.1.1 {

   283   sqlite3 db test.db

   284   execsql {

   285     PRAGMA page_size=1024;

   286     PRAGMA default_cache_size=10;

   287   }

   288   execsql {

   289     PRAGMA temp_store = memory;

   290     BEGIN;

   291     CREATE TABLE abc(a PRIMARY KEY, b, c);

   292     INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));

   293     INSERT INTO abc 

   294         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

   295     INSERT INTO abc 

   296         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

   297     INSERT INTO abc 

   298         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

   299     INSERT INTO abc 

   300         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

   301     INSERT INTO abc 

   302         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

   303     INSERT INTO abc 

   304         SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

   305     COMMIT;

   306   } 

   307   copy_file test.db test2.db

   308   sqlite3 db2 test2.db

   309   list \

   310     [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]

   311 } {1 1}

   312 do_test malloc5-6.1.2 {

   313   list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]

   314 } {10 10}

   315 

   316 do_test malloc5-6.2.1 {

   317 breakpoint

   318   execsql {SELECT * FROM abc} db2

   319   execsql {SELECT * FROM abc} db

   320   expr [nPage db] + [nPage db2]

   321 } {20}

   322 

   323 do_test malloc5-6.2.2 {

   324   # If we now try to reclaim some memory, it should come from the db2 cache.

   325   sqlite3_release_memory 3000

   326   expr [nPage db] + [nPage db2]

   327 } {17}

   328 do_test malloc5-6.2.3 {

   329   # Access the db2 cache again, so that all the db2 pages have been used

   330   # more recently than all the db pages. Then try to reclaim 3000 bytes.

   331   # This time, 3 pages should be pulled from the db cache.

   332   execsql { SELECT * FROM abc } db2

   333   sqlite3_release_memory 3000

   334   expr [nPage db] + [nPage db2]

   335 } {17}

   336 

   337 do_test malloc5-6.3.1 {

   338   # Now open a transaction and update 2 pages in the db2 cache. Then

   339   # do a SELECT on the db cache so that all the db pages are more recently

   340   # used than the db2 pages. When we try to free memory, SQLite should

   341   # free the non-dirty db2 pages, then the db pages, then finally use

   342   # sync() to free up the dirty db2 pages. The only page that cannot be

   343   # freed is page1 of db2. Because there is an open transaction, the

   344   # btree layer holds a reference to page 1 in the db2 cache.

   345   execsql {

   346     BEGIN;

   347     UPDATE abc SET c = randstr(100,100) 

   348     WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);

   349   } db2

   350   execsql { SELECT * FROM abc } db

   351   expr [nPage db] + [nPage db2]

   352 } {20}

   353 do_test malloc5-6.3.2 {

   354   # Try to release 7700 bytes. This should release all the 

   355   # non-dirty pages held by db2.

   356   sqlite3_release_memory [expr 7*1100]

   357   list [nPage db] [nPage db2]

   358 } {10 3}

   359 do_test malloc5-6.3.3 {

   360   # Try to release another 1000 bytes. This should come fromt the db

   361   # cache, since all three pages held by db2 are either in-use or diry.

   362   sqlite3_release_memory 1000

   363   list [nPage db] [nPage db2]

   364 } {9 3}

   365 do_test malloc5-6.3.4 {

   366   # Now release 9900 more (about 9 pages worth). This should expunge

   367   # the rest of the db cache. But the db2 cache remains intact, because

   368   # SQLite tries to avoid calling sync().

   369   sqlite3_release_memory 9900

   370   list [nPage db] [nPage db2]

   371 } {0 3}

   372 do_test malloc5-6.3.5 {

   373   # But if we are really insistent, SQLite will consent to call sync()

   374   # if there is no other option. UPDATE: As of 3.6.2, SQLite will not

   375   # call sync() in this scenario. So no further memory can be reclaimed.

   376   sqlite3_release_memory 1000

   377   list [nPage db] [nPage db2]

   378 } {0 3}

   379 do_test malloc5-6.3.6 {

   380   # The referenced page (page 1 of the db2 cache) will not be freed no

   381   # matter how much memory we ask for:

   382   sqlite3_release_memory 31459

   383   list [nPage db] [nPage db2]

   384 } {0 3}

   385 

   386 db2 close

   387 

   388 sqlite3_soft_heap_limit $::soft_limit

   389 finish_test

   390 catch {db close}