# 2005 November 30

#

# The author disclaims copyright to this source code.  In place of

# a legal notice, here is a blessing:

#

#    May you do good and not evil.

#    May you find forgiveness for yourself and forgive others.

#    May you share freely, never taking more than you give.

#

#***********************************************************************

#

# This file contains test cases focused on the two memory-management APIs, 

# sqlite3_soft_heap_limit() and sqlite3_release_memory().

#

# Prior to version 3.6.2, calling sqlite3_release_memory() or exceeding

# the configured soft heap limit could cause sqlite to upgrade database 

# locks and flush dirty pages to the file system. As of 3.6.2, this is

# no longer the case. In version 3.6.2, sqlite3_release_memory() only

# reclaims clean pages. This test file has been updated accordingly.

#

# $Id: malloc5.test,v 1.20 2008/08/27 16:38:57 danielk1977 Exp $

set testdir [file dirname $argv0]

source $testdir/tester.tcl

source $testdir/malloc_common.tcl

db close

# Only run these tests if memory debugging is turned on.

#

if {!$MEMDEBUG} {

   puts "Skipping malloc5 tests: not compiled with -DSQLITE_MEMDEBUG..."

   finish_test

   return

}

# Skip these tests if OMIT_MEMORY_MANAGEMENT was defined at compile time.

ifcapable !memorymanage {

   finish_test

   return

}

sqlite3_soft_heap_limit 0

sqlite3 db test.db

do_test malloc5-1.1 {

  # Simplest possible test. Call sqlite3_release_memory when there is exactly

  # one unused page in a single pager cache. The page cannot be freed, as

  # it is dirty. So sqlite3_release_memory() returns 0.

  #

  execsql {

    PRAGMA auto_vacuum=OFF;

    BEGIN;

    CREATE TABLE abc(a, b, c);

  }

  sqlite3_release_memory

} {0}

do_test malloc5-1.2 {

  # Test that the transaction started in the above test is still active.

  # The lock on the database file should not have been upgraded (this was

  # not the case before version 3.6.2).

  #

  sqlite3 db2 test.db

  execsql { SELECT * FROM sqlite_master } db2

} {}

do_test malloc5-1.3 {

  # Call [sqlite3_release_memory] when there is exactly one unused page 

  # in the cache belonging to db2.

  #

  set ::pgalloc [sqlite3_release_memory]

  expr $::pgalloc > 0

} {1}

do_test malloc5-1.4 {

  # Commit the transaction and open a new one. Read 1 page into the cache.

  # Because the page is not dirty, it is eligible for collection even

  # before the transaction is concluded.

  #

  execsql {

    COMMIT;

    BEGIN;

    SELECT * FROM abc;

  }

  sqlite3_release_memory

} $::pgalloc

do_test malloc5-1.5 {

  # Conclude the transaction opened in the previous [do_test] block. This

  # causes another page (page 1) to become eligible for recycling.

  #

  execsql { COMMIT }

  sqlite3_release_memory

} $::pgalloc

do_test malloc5-1.6 {

  # Manipulate the cache so that it contains two unused pages. One requires 

  # a journal-sync to free, the other does not.

  db2 close

  execsql {

    BEGIN;

    SELECT * FROM abc;

    CREATE TABLE def(d, e, f);

  }

  sqlite3_release_memory 500

} $::pgalloc

do_test malloc5-1.7 {

  # Database should not be locked this time. 

  sqlite3 db2 test.db

  catchsql { SELECT * FROM abc } db2

} {0 {}}

do_test malloc5-1.8 {

  # Try to release another block of memory. This will fail as the only

  # pages currently in the cache are dirty (page 3) or pinned (page 1).

  db2 close

  sqlite3_release_memory 500

} 0

do_test malloc5-1.8 {

  # Database is still not locked.

  #

  sqlite3 db2 test.db

  catchsql { SELECT * FROM abc } db2

} {0 {}}

do_test malloc5-1.9 {

  execsql {

    COMMIT;

  }

} {}

do_test malloc5-2.1 {

  # Put some data in tables abc and def. Both tables are still wholly 

  # contained within their root pages.

  execsql {

    INSERT INTO abc VALUES(1, 2, 3);

    INSERT INTO abc VALUES(4, 5, 6);

    INSERT INTO def VALUES(7, 8, 9);

    INSERT INTO def VALUES(10,11,12);

  }

} {}

do_test malloc5-2.2 {

  # Load the root-page for table def into the cache. Then query table abc. 

  # Halfway through the query call sqlite3_release_memory(). The goal of this

  # test is to make sure we don't free pages that are in use (specifically, 

  # the root of table abc).

  sqlite3_release_memory

  set nRelease 0

  execsql { 

    BEGIN;

    SELECT * FROM def;

  }

  set data [list]

  db eval {SELECT * FROM abc} {

    incr nRelease [sqlite3_release_memory]

    lappend data $a $b $c

  }

  execsql {

    COMMIT;

  }

  list $nRelease $data

} [list $pgalloc [list 1 2 3 4 5 6]]

do_test malloc5-3.1 {

  # Simple test to show that if two pagers are opened from within this

  # thread, memory is freed from both when sqlite3_release_memory() is

  # called.

  execsql {

    BEGIN;

    SELECT * FROM abc;

  }

  execsql {

    SELECT * FROM sqlite_master;

    BEGIN;

    SELECT * FROM def;

  } db2

  sqlite3_release_memory

} [expr $::pgalloc * 2]

do_test malloc5-3.2 {

  concat \

    [execsql {SELECT * FROM abc; COMMIT}] \

    [execsql {SELECT * FROM def; COMMIT} db2]

} {1 2 3 4 5 6 7 8 9 10 11 12}

db2 close

puts "Highwater mark: [sqlite3_memory_highwater]"

# The following two test cases each execute a transaction in which 

# 10000 rows are inserted into table abc. The first test case is used

# to ensure that more than 1MB of dynamic memory is used to perform

# the transaction. 

#

# The second test case sets the "soft-heap-limit" to 100,000 bytes (0.1 MB)

# and tests to see that this limit is not exceeded at any point during 

# transaction execution.

#

# Before executing malloc5-4.* we save the value of the current soft heap 

# limit in variable ::soft_limit. The original value is restored after 

# running the tests.

#

set ::soft_limit [sqlite3_soft_heap_limit -1]

execsql {PRAGMA cache_size=2000}

do_test malloc5-4.1 {

  execsql {BEGIN;}

  execsql {DELETE FROM abc;}

  for {set i 0} {$i < 10000} {incr i} {

    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"

  }

  execsql {COMMIT;}

  sqlite3_release_memory

  sqlite3_memory_highwater 1

  execsql {SELECT * FROM abc}

  set nMaxBytes [sqlite3_memory_highwater 1]

  puts -nonewline " (Highwater mark: $nMaxBytes) "

  expr $nMaxBytes > 1000000

} {1}

do_test malloc5-4.2 {

  sqlite3_release_memory

  sqlite3_soft_heap_limit 100000

  sqlite3_memory_highwater 1

  execsql {SELECT * FROM abc}

  set nMaxBytes [sqlite3_memory_highwater 1]

  puts -nonewline " (Highwater mark: $nMaxBytes) "

  expr $nMaxBytes <= 100000

} {1}

do_test malloc5-4.3 {

  # Check that the content of table abc is at least roughly as expected.

  execsql {

    SELECT count(*), sum(a), sum(b) FROM abc;

  }

} [list 10000 [expr int(10000.0 * 4999.5)] [expr int(10000.0 * 4999.5)]]

# Restore the soft heap limit.

sqlite3_soft_heap_limit $::soft_limit

# Test that there are no problems calling sqlite3_release_memory when

# there are open in-memory databases.

#

# At one point these tests would cause a seg-fault.

#

do_test malloc5-5.1 {

  db close

  sqlite3 db :memory:

  execsql {

    BEGIN;

    CREATE TABLE abc(a, b, c);

    INSERT INTO abc VALUES('abcdefghi', 1234567890, NULL);

    INSERT INTO abc SELECT * FROM abc;

    INSERT INTO abc SELECT * FROM abc;

    INSERT INTO abc SELECT * FROM abc;

    INSERT INTO abc SELECT * FROM abc;

    INSERT INTO abc SELECT * FROM abc;

    INSERT INTO abc SELECT * FROM abc;

    INSERT INTO abc SELECT * FROM abc;

  }

  sqlite3_release_memory

} 0

do_test malloc5-5.2 {

  sqlite3_soft_heap_limit 5000

  execsql {

    COMMIT;

    PRAGMA temp_store = memory;

    SELECT * FROM abc ORDER BY a;

  }

  expr 1

} {1}

sqlite3_soft_heap_limit $::soft_limit

#-------------------------------------------------------------------------

# The following test cases (malloc5-6.*) test the new global LRU list

# used to determine the pages to recycle when sqlite3_release_memory is

# called and there is more than one pager open.

#

proc nPage {db} {

  set bt [btree_from_db $db]

  array set stats [btree_pager_stats $bt]

  set stats(page)

}

db close

file delete -force test.db test.db-journal test2.db test2.db-journal

# This block of test-cases (malloc5-6.1.*) prepares two database files

# for the subsequent tests.

do_test malloc5-6.1.1 {

  sqlite3 db test.db

  execsql {

    PRAGMA page_size=1024;

    PRAGMA default_cache_size=10;

  }

  execsql {

    PRAGMA temp_store = memory;

    BEGIN;

    CREATE TABLE abc(a PRIMARY KEY, b, c);

    INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));

    INSERT INTO abc 

        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

    INSERT INTO abc 

        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

    INSERT INTO abc 

        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

    INSERT INTO abc 

        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

    INSERT INTO abc 

        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

    INSERT INTO abc 

        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

    COMMIT;

  } 

  copy_file test.db test2.db

  sqlite3 db2 test2.db

  list \

    [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]

} {1 1}

do_test malloc5-6.1.2 {

  list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]

} {10 10}

do_test malloc5-6.2.1 {

breakpoint

  execsql {SELECT * FROM abc} db2

  execsql {SELECT * FROM abc} db

  expr [nPage db] + [nPage db2]

} {20}

do_test malloc5-6.2.2 {

  # If we now try to reclaim some memory, it should come from the db2 cache.

  sqlite3_release_memory 3000

  expr [nPage db] + [nPage db2]

} {17}

do_test malloc5-6.2.3 {

  # Access the db2 cache again, so that all the db2 pages have been used

  # more recently than all the db pages. Then try to reclaim 3000 bytes.

  # This time, 3 pages should be pulled from the db cache.

  execsql { SELECT * FROM abc } db2

  sqlite3_release_memory 3000

  expr [nPage db] + [nPage db2]

} {17}

do_test malloc5-6.3.1 {

  # Now open a transaction and update 2 pages in the db2 cache. Then

  # do a SELECT on the db cache so that all the db pages are more recently

  # used than the db2 pages. When we try to free memory, SQLite should

  # free the non-dirty db2 pages, then the db pages, then finally use

  # sync() to free up the dirty db2 pages. The only page that cannot be

  # freed is page1 of db2. Because there is an open transaction, the

  # btree layer holds a reference to page 1 in the db2 cache.

  execsql {

    BEGIN;

    UPDATE abc SET c = randstr(100,100) 

    WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);

  } db2

  execsql { SELECT * FROM abc } db

  expr [nPage db] + [nPage db2]

} {20}

do_test malloc5-6.3.2 {

  # Try to release 7700 bytes. This should release all the 

  # non-dirty pages held by db2.

  sqlite3_release_memory [expr 7*1100]

  list [nPage db] [nPage db2]

} {10 3}

do_test malloc5-6.3.3 {

  # Try to release another 1000 bytes. This should come fromt the db

  # cache, since all three pages held by db2 are either in-use or diry.

  sqlite3_release_memory 1000

  list [nPage db] [nPage db2]

} {9 3}

do_test malloc5-6.3.4 {

  # Now release 9900 more (about 9 pages worth). This should expunge

  # the rest of the db cache. But the db2 cache remains intact, because

  # SQLite tries to avoid calling sync().

  sqlite3_release_memory 9900

  list [nPage db] [nPage db2]

} {0 3}

do_test malloc5-6.3.5 {

  # But if we are really insistent, SQLite will consent to call sync()

  # if there is no other option. UPDATE: As of 3.6.2, SQLite will not

  # call sync() in this scenario. So no further memory can be reclaimed.

  sqlite3_release_memory 1000

  list [nPage db] [nPage db2]

} {0 3}

do_test malloc5-6.3.6 {

  # The referenced page (page 1 of the db2 cache) will not be freed no

  # matter how much memory we ask for:

  sqlite3_release_memory 31459

  list [nPage db] [nPage db2]

} {0 3}

db2 close

sqlite3_soft_heap_limit $::soft_limit

finish_test

catch {db close}