1 /*

     2 ** 2004 May 22

     3 **

     4 ** The author disclaims copyright to this source code.  In place of

     5 ** a legal notice, here is a blessing:

     6 **

     7 **    May you do good and not evil.

     8 **    May you find forgiveness for yourself and forgive others.

     9 **    May you share freely, never taking more than you give.

    10 **

    11 ******************************************************************************

    12 **

    13 ** This file contains code that is specific to Unix systems.

    14 */

    15 #include "sqliteInt.h"

    16 #include "os.h"

    17 #if OS_UNIX              /* This file is used on unix only */

    18 

    19 /*

    20 ** These #defines should enable >2GB file support on Posix if the

    21 ** underlying operating system supports it.  If the OS lacks

    22 ** large file support, these should be no-ops.

    23 **

    24 ** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch

    25 ** on the compiler command line.  This is necessary if you are compiling

    26 ** on a recent machine (ex: RedHat 7.2) but you want your code to work

    27 ** on an older machine (ex: RedHat 6.0).  If you compile on RedHat 7.2

    28 ** without this option, LFS is enable.  But LFS does not exist in the kernel

    29 ** in RedHat 6.0, so the code won't work.  Hence, for maximum binary

    30 ** portability you should omit LFS.

    31 */

    32 #ifndef SQLITE_DISABLE_LFS

    33 # define _LARGE_FILE       1

    34 # ifndef _FILE_OFFSET_BITS

    35 #   define _FILE_OFFSET_BITS 64

    36 # endif

    37 # define _LARGEFILE_SOURCE 1

    38 #endif

    39 

    40 /*

    41 ** standard include files.

    42 */

    43 #include <sys/types.h>

    44 #include <sys/stat.h>

    45 #include <fcntl.h>

    46 #include <unistd.h>

    47 #include <time.h>

    48 #include <sys/time.h>

    49 #include <errno.h>

    50 

    51 /*

    52 ** If we are to be thread-safe, include the pthreads header and define

    53 ** the SQLITE_UNIX_THREADS macro.

    54 */

    55 #if defined(THREADSAFE) && THREADSAFE

    56 # include <pthread.h>

    57 # define SQLITE_UNIX_THREADS 1

    58 #endif

    59 

    60 /*

    61 ** Default permissions when creating a new file

    62 */

    63 #ifndef SQLITE_DEFAULT_FILE_PERMISSIONS

    64 # define SQLITE_DEFAULT_FILE_PERMISSIONS 0644

    65 #endif

    66 

    67 

    68 

    69 /*

    70 ** The unixFile structure is subclass of OsFile specific for the unix

    71 ** protability layer.

    72 */

    73 typedef struct unixFile unixFile;

    74 struct unixFile {

    75   IoMethod const *pMethod;  /* Always the first entry */

    76   struct openCnt *pOpen;    /* Info about all open fd's on this inode */

    77   struct lockInfo *pLock;   /* Info about locks on this inode */

    78   int h;                    /* The file descriptor */

    79   unsigned char locktype;   /* The type of lock held on this fd */

    80   unsigned char isOpen;     /* True if needs to be closed */

    81   unsigned char fullSync;   /* Use F_FULLSYNC if available */

    82   int dirfd;                /* File descriptor for the directory */

    83   i64 offset;               /* Seek offset */

    84 #ifdef SQLITE_UNIX_THREADS

    85   pthread_t tid;            /* The thread that "owns" this OsFile */

    86 #endif

    87 };

    88 

    89 /*

    90 ** Provide the ability to override some OS-layer functions during

    91 ** testing.  This is used to simulate OS crashes to verify that 

    92 ** commits are atomic even in the event of an OS crash.

    93 */

    94 #ifdef SQLITE_CRASH_TEST

    95   extern int sqlite3CrashTestEnable;

    96   extern int sqlite3CrashOpenReadWrite(const char*, OsFile**, int*);

    97   extern int sqlite3CrashOpenExclusive(const char*, OsFile**, int);

    98   extern int sqlite3CrashOpenReadOnly(const char*, OsFile**, int);

    99 # define CRASH_TEST_OVERRIDE(X,A,B,C) \

   100     if(sqlite3CrashTestEnable){ return X(A,B,C); }

   101 #else

   102 # define CRASH_TEST_OVERRIDE(X,A,B,C)  /* no-op */

   103 #endif

   104 

   105 

   106 /*

   107 ** Include code that is common to all os_*.c files

   108 */

   109 #include "os_common.h"

   110 

   111 /*

   112 ** Do not include any of the File I/O interface procedures if the

   113 ** SQLITE_OMIT_DISKIO macro is defined (indicating that the database

   114 ** will be in-memory only)

   115 */

   116 #ifndef SQLITE_OMIT_DISKIO

   117 

   118 

   119 /*

   120 ** Define various macros that are missing from some systems.

   121 */

   122 #ifndef O_LARGEFILE

   123 # define O_LARGEFILE 0

   124 #endif

   125 #ifdef SQLITE_DISABLE_LFS

   126 # undef O_LARGEFILE

   127 # define O_LARGEFILE 0

   128 #endif

   129 #ifndef O_NOFOLLOW

   130 # define O_NOFOLLOW 0

   131 #endif

   132 #ifndef O_BINARY

   133 # define O_BINARY 0

   134 #endif

   135 

   136 /*

   137 ** The DJGPP compiler environment looks mostly like Unix, but it

   138 ** lacks the fcntl() system call.  So redefine fcntl() to be something

   139 ** that always succeeds.  This means that locking does not occur under

   140 ** DJGPP.  But it's DOS - what did you expect?

   141 */

   142 #ifdef __DJGPP__

   143 # define fcntl(A,B,C) 0

   144 #endif

   145 

   146 /*

   147 ** The threadid macro resolves to the thread-id or to 0.  Used for

   148 ** testing and debugging only.

   149 */

   150 #ifdef SQLITE_UNIX_THREADS

   151 #define threadid pthread_self()

   152 #else

   153 #define threadid 0

   154 #endif

   155 

   156 /*

   157 ** Set or check the OsFile.tid field.  This field is set when an OsFile

   158 ** is first opened.  All subsequent uses of the OsFile verify that the

   159 ** same thread is operating on the OsFile.  Some operating systems do

   160 ** not allow locks to be overridden by other threads and that restriction

   161 ** means that sqlite3* database handles cannot be moved from one thread

   162 ** to another.  This logic makes sure a user does not try to do that

   163 ** by mistake.

   164 **

   165 ** Version 3.3.1 (2006-01-15):  OsFiles can be moved from one thread to

   166 ** another as long as we are running on a system that supports threads

   167 ** overriding each others locks (which now the most common behavior)

   168 ** or if no locks are held.  But the OsFile.pLock field needs to be

   169 ** recomputed because its key includes the thread-id.  See the 

   170 ** transferOwnership() function below for additional information

   171 */

   172 #if defined(SQLITE_UNIX_THREADS)

   173 # define SET_THREADID(X)   (X)->tid = pthread_self()

   174 # define CHECK_THREADID(X) (threadsOverrideEachOthersLocks==0 && \

   175                             !pthread_equal((X)->tid, pthread_self()))

   176 #else

   177 # define SET_THREADID(X)

   178 # define CHECK_THREADID(X) 0

   179 #endif

   180 

   181 /*

   182 ** Here is the dirt on POSIX advisory locks:  ANSI STD 1003.1 (1996)

   183 ** section 6.5.2.2 lines 483 through 490 specify that when a process

   184 ** sets or clears a lock, that operation overrides any prior locks set

   185 ** by the same process.  It does not explicitly say so, but this implies

   186 ** that it overrides locks set by the same process using a different

   187 ** file descriptor.  Consider this test case:

   188 **

   189 **       int fd1 = open("./file1", O_RDWR|O_CREAT, 0644);

   190 **       int fd2 = open("./file2", O_RDWR|O_CREAT, 0644);

   191 **

   192 ** Suppose ./file1 and ./file2 are really the same file (because

   193 ** one is a hard or symbolic link to the other) then if you set

   194 ** an exclusive lock on fd1, then try to get an exclusive lock

   195 ** on fd2, it works.  I would have expected the second lock to

   196 ** fail since there was already a lock on the file due to fd1.

   197 ** But not so.  Since both locks came from the same process, the

   198 ** second overrides the first, even though they were on different

   199 ** file descriptors opened on different file names.

   200 **

   201 ** Bummer.  If you ask me, this is broken.  Badly broken.  It means

   202 ** that we cannot use POSIX locks to synchronize file access among

   203 ** competing threads of the same process.  POSIX locks will work fine

   204 ** to synchronize access for threads in separate processes, but not

   205 ** threads within the same process.

   206 **

   207 ** To work around the problem, SQLite has to manage file locks internally

   208 ** on its own.  Whenever a new database is opened, we have to find the

   209 ** specific inode of the database file (the inode is determined by the

   210 ** st_dev and st_ino fields of the stat structure that fstat() fills in)

   211 ** and check for locks already existing on that inode.  When locks are

   212 ** created or removed, we have to look at our own internal record of the

   213 ** locks to see if another thread has previously set a lock on that same

   214 ** inode.

   215 **

   216 ** The OsFile structure for POSIX is no longer just an integer file

   217 ** descriptor.  It is now a structure that holds the integer file

   218 ** descriptor and a pointer to a structure that describes the internal

   219 ** locks on the corresponding inode.  There is one locking structure

   220 ** per inode, so if the same inode is opened twice, both OsFile structures

   221 ** point to the same locking structure.  The locking structure keeps

   222 ** a reference count (so we will know when to delete it) and a "cnt"

   223 ** field that tells us its internal lock status.  cnt==0 means the

   224 ** file is unlocked.  cnt==-1 means the file has an exclusive lock.

   225 ** cnt>0 means there are cnt shared locks on the file.

   226 **

   227 ** Any attempt to lock or unlock a file first checks the locking

   228 ** structure.  The fcntl() system call is only invoked to set a 

   229 ** POSIX lock if the internal lock structure transitions between

   230 ** a locked and an unlocked state.

   231 **

   232 ** 2004-Jan-11:

   233 ** More recent discoveries about POSIX advisory locks.  (The more

   234 ** I discover, the more I realize the a POSIX advisory locks are

   235 ** an abomination.)

   236 **

   237 ** If you close a file descriptor that points to a file that has locks,

   238 ** all locks on that file that are owned by the current process are

   239 ** released.  To work around this problem, each OsFile structure contains

   240 ** a pointer to an openCnt structure.  There is one openCnt structure

   241 ** per open inode, which means that multiple OsFiles can point to a single

   242 ** openCnt.  When an attempt is made to close an OsFile, if there are

   243 ** other OsFiles open on the same inode that are holding locks, the call

   244 ** to close() the file descriptor is deferred until all of the locks clear.

   245 ** The openCnt structure keeps a list of file descriptors that need to

   246 ** be closed and that list is walked (and cleared) when the last lock

   247 ** clears.

   248 **

   249 ** First, under Linux threads, because each thread has a separate

   250 ** process ID, lock operations in one thread do not override locks

   251 ** to the same file in other threads.  Linux threads behave like

   252 ** separate processes in this respect.  But, if you close a file

   253 ** descriptor in linux threads, all locks are cleared, even locks

   254 ** on other threads and even though the other threads have different

   255 ** process IDs.  Linux threads is inconsistent in this respect.

   256 ** (I'm beginning to think that linux threads is an abomination too.)

   257 ** The consequence of this all is that the hash table for the lockInfo

   258 ** structure has to include the process id as part of its key because

   259 ** locks in different threads are treated as distinct.  But the 

   260 ** openCnt structure should not include the process id in its

   261 ** key because close() clears lock on all threads, not just the current

   262 ** thread.  Were it not for this goofiness in linux threads, we could

   263 ** combine the lockInfo and openCnt structures into a single structure.

   264 **

   265 ** 2004-Jun-28:

   266 ** On some versions of linux, threads can override each others locks.

   267 ** On others not.  Sometimes you can change the behavior on the same

   268 ** system by setting the LD_ASSUME_KERNEL environment variable.  The

   269 ** POSIX standard is silent as to which behavior is correct, as far

   270 ** as I can tell, so other versions of unix might show the same

   271 ** inconsistency.  There is no little doubt in my mind that posix

   272 ** advisory locks and linux threads are profoundly broken.

   273 **

   274 ** To work around the inconsistencies, we have to test at runtime 

   275 ** whether or not threads can override each others locks.  This test

   276 ** is run once, the first time any lock is attempted.  A static 

   277 ** variable is set to record the results of this test for future

   278 ** use.

   279 */

   280 

   281 /*

   282 ** An instance of the following structure serves as the key used

   283 ** to locate a particular lockInfo structure given its inode.

   284 **

   285 ** If threads cannot override each others locks, then we set the

   286 ** lockKey.tid field to the thread ID.  If threads can override

   287 ** each others locks then tid is always set to zero.  tid is omitted

   288 ** if we compile without threading support.

   289 */

   290 struct lockKey {

   291   dev_t dev;       /* Device number */

   292   ino_t ino;       /* Inode number */

   293 #ifdef SQLITE_UNIX_THREADS

   294   pthread_t tid;   /* Thread ID or zero if threads can override each other */

   295 #endif

   296 };

   297 

   298 /*

   299 ** An instance of the following structure is allocated for each open

   300 ** inode on each thread with a different process ID.  (Threads have

   301 ** different process IDs on linux, but not on most other unixes.)

   302 **

   303 ** A single inode can have multiple file descriptors, so each OsFile

   304 ** structure contains a pointer to an instance of this object and this

   305 ** object keeps a count of the number of OsFiles pointing to it.

   306 */

   307 struct lockInfo {

   308   struct lockKey key;  /* The lookup key */

   309   int cnt;             /* Number of SHARED locks held */

   310   int locktype;        /* One of SHARED_LOCK, RESERVED_LOCK etc. */

   311   int nRef;            /* Number of pointers to this structure */

   312 };

   313 

   314 /*

   315 ** An instance of the following structure serves as the key used

   316 ** to locate a particular openCnt structure given its inode.  This

   317 ** is the same as the lockKey except that the thread ID is omitted.

   318 */

   319 struct openKey {

   320   dev_t dev;   /* Device number */

   321   ino_t ino;   /* Inode number */

   322 };

   323 

   324 /*

   325 ** An instance of the following structure is allocated for each open

   326 ** inode.  This structure keeps track of the number of locks on that

   327 ** inode.  If a close is attempted against an inode that is holding

   328 ** locks, the close is deferred until all locks clear by adding the

   329 ** file descriptor to be closed to the pending list.

   330 */

   331 struct openCnt {

   332   struct openKey key;   /* The lookup key */

   333   int nRef;             /* Number of pointers to this structure */

   334   int nLock;            /* Number of outstanding locks */

   335   int nPending;         /* Number of pending close() operations */

   336   int *aPending;        /* Malloced space holding fd's awaiting a close() */

   337 };

   338 

   339 /* 

   340 ** These hash tables map inodes and file descriptors (really, lockKey and

   341 ** openKey structures) into lockInfo and openCnt structures.  Access to 

   342 ** these hash tables must be protected by a mutex.

   343 */

   344 static Hash lockHash = {SQLITE_HASH_BINARY, 0, 0, 0, 

   345     sqlite3ThreadSafeMalloc, sqlite3ThreadSafeFree, 0, 0};

   346 static Hash openHash = {SQLITE_HASH_BINARY, 0, 0, 0, 

   347     sqlite3ThreadSafeMalloc, sqlite3ThreadSafeFree, 0, 0};

   348 

   349 #ifdef SQLITE_UNIX_THREADS

   350 /*

   351 ** This variable records whether or not threads can override each others

   352 ** locks.

   353 **

   354 **    0:  No.  Threads cannot override each others locks.

   355 **    1:  Yes.  Threads can override each others locks.

   356 **   -1:  We don't know yet.

   357 **

   358 ** On some systems, we know at compile-time if threads can override each

   359 ** others locks.  On those systems, the SQLITE_THREAD_OVERRIDE_LOCK macro

   360 ** will be set appropriately.  On other systems, we have to check at

   361 ** runtime.  On these latter systems, SQLTIE_THREAD_OVERRIDE_LOCK is

   362 ** undefined.

   363 **

   364 ** This variable normally has file scope only.  But during testing, we make

   365 ** it a global so that the test code can change its value in order to verify

   366 ** that the right stuff happens in either case.

   367 */

   368 #ifndef SQLITE_THREAD_OVERRIDE_LOCK

   369 # define SQLITE_THREAD_OVERRIDE_LOCK -1

   370 #endif

   371 #ifdef SQLITE_TEST

   372 int threadsOverrideEachOthersLocks = SQLITE_THREAD_OVERRIDE_LOCK;

   373 #else

   374 static int threadsOverrideEachOthersLocks = SQLITE_THREAD_OVERRIDE_LOCK;

   375 #endif

   376 

   377 /*

   378 ** This structure holds information passed into individual test

   379 ** threads by the testThreadLockingBehavior() routine.

   380 */

   381 struct threadTestData {

   382   int fd;                /* File to be locked */

   383   struct flock lock;     /* The locking operation */

   384   int result;            /* Result of the locking operation */

   385 };

   386 

   387 #ifdef SQLITE_LOCK_TRACE

   388 /*

   389 ** Print out information about all locking operations.

   390 **

   391 ** This routine is used for troubleshooting locks on multithreaded

   392 ** platforms.  Enable by compiling with the -DSQLITE_LOCK_TRACE

   393 ** command-line option on the compiler.  This code is normally

   394 ** turned off.

   395 */

   396 static int lockTrace(int fd, int op, struct flock *p){

   397   char *zOpName, *zType;

   398   int s;

   399   int savedErrno;

   400   if( op==F_GETLK ){

   401     zOpName = "GETLK";

   402   }else if( op==F_SETLK ){

   403     zOpName = "SETLK";

   404   }else{

   405     s = fcntl(fd, op, p);

   406     sqlite3DebugPrintf("fcntl unknown %d %d %d\n", fd, op, s);

   407     return s;

   408   }

   409   if( p->l_type==F_RDLCK ){

   410     zType = "RDLCK";

   411   }else if( p->l_type==F_WRLCK ){

   412     zType = "WRLCK";

   413   }else if( p->l_type==F_UNLCK ){

   414     zType = "UNLCK";

   415   }else{

   416     assert( 0 );

   417   }

   418   assert( p->l_whence==SEEK_SET );

   419   s = fcntl(fd, op, p);

   420   savedErrno = errno;

   421   sqlite3DebugPrintf("fcntl %d %d %s %s %d %d %d %d\n",

   422      threadid, fd, zOpName, zType, (int)p->l_start, (int)p->l_len,

   423      (int)p->l_pid, s);

   424   if( s && op==F_SETLK && (p->l_type==F_RDLCK || p->l_type==F_WRLCK) ){

   425     struct flock l2;

   426     l2 = *p;

   427     fcntl(fd, F_GETLK, &l2);

   428     if( l2.l_type==F_RDLCK ){

   429       zType = "RDLCK";

   430     }else if( l2.l_type==F_WRLCK ){

   431       zType = "WRLCK";

   432     }else if( l2.l_type==F_UNLCK ){

   433       zType = "UNLCK";

   434     }else{

   435       assert( 0 );

   436     }

   437     sqlite3DebugPrintf("fcntl-failure-reason: %s %d %d %d\n",

   438        zType, (int)l2.l_start, (int)l2.l_len, (int)l2.l_pid);

   439   }

   440   errno = savedErrno;

   441   return s;

   442 }

   443 #define fcntl lockTrace

   444 #endif /* SQLITE_LOCK_TRACE */

   445 

   446 /*

   447 ** The testThreadLockingBehavior() routine launches two separate

   448 ** threads on this routine.  This routine attempts to lock a file

   449 ** descriptor then returns.  The success or failure of that attempt

   450 ** allows the testThreadLockingBehavior() procedure to determine

   451 ** whether or not threads can override each others locks.

   452 */

   453 static void *threadLockingTest(void *pArg){

   454   struct threadTestData *pData = (struct threadTestData*)pArg;

   455   pData->result = fcntl(pData->fd, F_SETLK, &pData->lock);

   456   return pArg;

   457 }

   458 

   459 /*

   460 ** This procedure attempts to determine whether or not threads

   461 ** can override each others locks then sets the 

   462 ** threadsOverrideEachOthersLocks variable appropriately.

   463 */

   464 static void testThreadLockingBehavior(int fd_orig){

   465   int fd;

   466   struct threadTestData d[2];

   467   pthread_t t[2];

   468 

   469   fd = dup(fd_orig);

   470   if( fd<0 ) return;

   471   memset(d, 0, sizeof(d));

   472   d[0].fd = fd;

   473   d[0].lock.l_type = F_RDLCK;

   474   d[0].lock.l_len = 1;

   475   d[0].lock.l_start = 0;

   476   d[0].lock.l_whence = SEEK_SET;

   477   d[1] = d[0];

   478   d[1].lock.l_type = F_WRLCK;

   479   pthread_create(&t[0], 0, threadLockingTest, &d[0]);

   480   pthread_create(&t[1], 0, threadLockingTest, &d[1]);

   481   pthread_join(t[0], 0);

   482   pthread_join(t[1], 0);

   483   close(fd);

   484   threadsOverrideEachOthersLocks =  d[0].result==0 && d[1].result==0;

   485 }

   486 #endif /* SQLITE_UNIX_THREADS */

   487 

   488 /*

   489 ** Release a lockInfo structure previously allocated by findLockInfo().

   490 */

   491 static void releaseLockInfo(struct lockInfo *pLock){

   492   assert( sqlite3OsInMutex(1) );

   493   pLock->nRef--;

   494   if( pLock->nRef==0 ){

   495     sqlite3HashInsert(&lockHash, &pLock->key, sizeof(pLock->key), 0);

   496     sqlite3ThreadSafeFree(pLock);

   497   }

   498 }

   499 

   500 /*

   501 ** Release a openCnt structure previously allocated by findLockInfo().

   502 */

   503 static void releaseOpenCnt(struct openCnt *pOpen){

   504   assert( sqlite3OsInMutex(1) );

   505   pOpen->nRef--;

   506   if( pOpen->nRef==0 ){

   507     sqlite3HashInsert(&openHash, &pOpen->key, sizeof(pOpen->key), 0);

   508     free(pOpen->aPending);

   509     sqlite3ThreadSafeFree(pOpen);

   510   }

   511 }

   512 

   513 /*

   514 ** Given a file descriptor, locate lockInfo and openCnt structures that

   515 ** describes that file descriptor.  Create new ones if necessary.  The

   516 ** return values might be uninitialized if an error occurs.

   517 **

   518 ** Return the number of errors.

   519 */

   520 static int findLockInfo(

   521   int fd,                      /* The file descriptor used in the key */

   522   struct lockInfo **ppLock,    /* Return the lockInfo structure here */

   523   struct openCnt **ppOpen      /* Return the openCnt structure here */

   524 ){

   525   int rc;

   526   struct lockKey key1;

   527   struct openKey key2;

   528   struct stat statbuf;

   529   struct lockInfo *pLock;

   530   struct openCnt *pOpen;

   531   rc = fstat(fd, &statbuf);

   532   if( rc!=0 ) return 1;

   533 

   534   assert( sqlite3OsInMutex(1) );

   535   memset(&key1, 0, sizeof(key1));

   536   key1.dev = statbuf.st_dev;

   537   key1.ino = statbuf.st_ino;

   538 #ifdef SQLITE_UNIX_THREADS

   539   if( threadsOverrideEachOthersLocks<0 ){

   540     testThreadLockingBehavior(fd);

   541   }

   542   key1.tid = threadsOverrideEachOthersLocks ? 0 : pthread_self();

   543 #endif

   544   memset(&key2, 0, sizeof(key2));

   545   key2.dev = statbuf.st_dev;

   546   key2.ino = statbuf.st_ino;

   547   pLock = (struct lockInfo*)sqlite3HashFind(&lockHash, &key1, sizeof(key1));

   548   if( pLock==0 ){

   549     struct lockInfo *pOld;

   550     pLock = sqlite3ThreadSafeMalloc( sizeof(*pLock) );

   551     if( pLock==0 ){

   552       rc = 1;

   553       goto exit_findlockinfo;

   554     }

   555     pLock->key = key1;

   556     pLock->nRef = 1;

   557     pLock->cnt = 0;

   558     pLock->locktype = 0;

   559     pOld = sqlite3HashInsert(&lockHash, &pLock->key, sizeof(key1), pLock);

   560     if( pOld!=0 ){

   561       assert( pOld==pLock );

   562       sqlite3ThreadSafeFree(pLock);

   563       rc = 1;

   564       goto exit_findlockinfo;

   565     }

   566   }else{

   567     pLock->nRef++;

   568   }

   569   *ppLock = pLock;

   570   if( ppOpen!=0 ){

   571     pOpen = (struct openCnt*)sqlite3HashFind(&openHash, &key2, sizeof(key2));

   572     if( pOpen==0 ){

   573       struct openCnt *pOld;

   574       pOpen = sqlite3ThreadSafeMalloc( sizeof(*pOpen) );

   575       if( pOpen==0 ){

   576         releaseLockInfo(pLock);

   577         rc = 1;

   578         goto exit_findlockinfo;

   579       }

   580       pOpen->key = key2;

   581       pOpen->nRef = 1;

   582       pOpen->nLock = 0;

   583       pOpen->nPending = 0;

   584       pOpen->aPending = 0;

   585       pOld = sqlite3HashInsert(&openHash, &pOpen->key, sizeof(key2), pOpen);

   586       if( pOld!=0 ){

   587         assert( pOld==pOpen );

   588         sqlite3ThreadSafeFree(pOpen);

   589         releaseLockInfo(pLock);

   590         rc = 1;

   591         goto exit_findlockinfo;

   592       }

   593     }else{

   594       pOpen->nRef++;

   595     }

   596     *ppOpen = pOpen;

   597   }

   598 

   599 exit_findlockinfo:

   600   return rc;

   601 }

   602 

   603 #ifdef SQLITE_DEBUG

   604 /*

   605 ** Helper function for printing out trace information from debugging

   606 ** binaries. This returns the string represetation of the supplied

   607 ** integer lock-type.

   608 */

   609 static const char *locktypeName(int locktype){

   610   switch( locktype ){

   611   case NO_LOCK: return "NONE";

   612   case SHARED_LOCK: return "SHARED";

   613   case RESERVED_LOCK: return "RESERVED";

   614   case PENDING_LOCK: return "PENDING";

   615   case EXCLUSIVE_LOCK: return "EXCLUSIVE";

   616   }

   617   return "ERROR";

   618 }

   619 #endif

   620 

   621 /*

   622 ** If we are currently in a different thread than the thread that the

   623 ** unixFile argument belongs to, then transfer ownership of the unixFile

   624 ** over to the current thread.

   625 **

   626 ** A unixFile is only owned by a thread on systems where one thread is

   627 ** unable to override locks created by a different thread.  RedHat9 is

   628 ** an example of such a system.

   629 **

   630 ** Ownership transfer is only allowed if the unixFile is currently unlocked.

   631 ** If the unixFile is locked and an ownership is wrong, then return

   632 ** SQLITE_MISUSE.  SQLITE_OK is returned if everything works.

   633 */

   634 #ifdef SQLITE_UNIX_THREADS

   635 static int transferOwnership(unixFile *pFile){

   636   int rc;

   637   pthread_t hSelf;

   638   if( threadsOverrideEachOthersLocks ){

   639     /* Ownership transfers not needed on this system */

   640     return SQLITE_OK;

   641   }

   642   hSelf = pthread_self();

   643   if( pthread_equal(pFile->tid, hSelf) ){

   644     /* We are still in the same thread */

   645     TRACE1("No-transfer, same thread\n");

   646     return SQLITE_OK;

   647   }

   648   if( pFile->locktype!=NO_LOCK ){

   649     /* We cannot change ownership while we are holding a lock! */

   650     return SQLITE_MISUSE;

   651   }

   652   TRACE4("Transfer ownership of %d from %d to %d\n", pFile->h,pFile->tid,hSelf);

   653   pFile->tid = hSelf;

   654   releaseLockInfo(pFile->pLock);

   655   rc = findLockInfo(pFile->h, &pFile->pLock, 0);

   656   TRACE5("LOCK    %d is now %s(%s,%d)\n", pFile->h,

   657      locktypeName(pFile->locktype),

   658      locktypeName(pFile->pLock->locktype), pFile->pLock->cnt);

   659   return rc;

   660 }

   661 #else

   662   /* On single-threaded builds, ownership transfer is a no-op */

   663 # define transferOwnership(X) SQLITE_OK

   664 #endif

   665 

   666 /*

   667 ** Delete the named file

   668 */

   669 int sqlite3UnixDelete(const char *zFilename){

   670   unlink(zFilename);

   671   return SQLITE_OK;

   672 }

   673 

   674 /*

   675 ** Return TRUE if the named file exists.

   676 */

   677 int sqlite3UnixFileExists(const char *zFilename){

   678   return access(zFilename, 0)==0;

   679 }

   680 

   681 /* Forward declaration */

   682 static int allocateUnixFile(unixFile *pInit, OsFile **pId);

   683 

   684 /*

   685 ** Attempt to open a file for both reading and writing.  If that

   686 ** fails, try opening it read-only.  If the file does not exist,

   687 ** try to create it.

   688 **

   689 ** On success, a handle for the open file is written to *id

   690 ** and *pReadonly is set to 0 if the file was opened for reading and

   691 ** writing or 1 if the file was opened read-only.  The function returns

   692 ** SQLITE_OK.

   693 **

   694 ** On failure, the function returns SQLITE_CANTOPEN and leaves

   695 ** *id and *pReadonly unchanged.

   696 */

   697 int sqlite3UnixOpenReadWrite(

   698   const char *zFilename,

   699   OsFile **pId,

   700   int *pReadonly

   701 ){

   702   int rc;

   703   unixFile f;

   704 

   705   CRASH_TEST_OVERRIDE(sqlite3CrashOpenReadWrite, zFilename, pId, pReadonly);

   706   assert( 0==*pId );

   707   f.h = open(zFilename, O_RDWR|O_CREAT|O_LARGEFILE|O_BINARY,

   708                           SQLITE_DEFAULT_FILE_PERMISSIONS);

   709   if( f.h<0 ){

   710 #ifdef EISDIR

   711     if( errno==EISDIR ){

   712       return SQLITE_CANTOPEN;

   713     }

   714 #endif

   715     f.h = open(zFilename, O_RDONLY|O_LARGEFILE|O_BINARY);

   716     if( f.h<0 ){

   717       return SQLITE_CANTOPEN; 

   718     }

   719     *pReadonly = 1;

   720   }else{

   721     *pReadonly = 0;

   722   }

   723   sqlite3OsEnterMutex();

   724   rc = findLockInfo(f.h, &f.pLock, &f.pOpen);

   725   sqlite3OsLeaveMutex();

   726   if( rc ){

   727     close(f.h);

   728     return SQLITE_NOMEM;

   729   }

   730   TRACE3("OPEN    %-3d %s\n", f.h, zFilename);

   731   return allocateUnixFile(&f, pId);

   732 }

   733 

   734 

   735 /*

   736 ** Attempt to open a new file for exclusive access by this process.

   737 ** The file will be opened for both reading and writing.  To avoid

   738 ** a potential security problem, we do not allow the file to have

   739 ** previously existed.  Nor do we allow the file to be a symbolic

   740 ** link.

   741 **

   742 ** If delFlag is true, then make arrangements to automatically delete

   743 ** the file when it is closed.

   744 **

   745 ** On success, write the file handle into *id and return SQLITE_OK.

   746 **

   747 ** On failure, return SQLITE_CANTOPEN.

   748 */

   749 int sqlite3UnixOpenExclusive(const char *zFilename, OsFile **pId, int delFlag){

   750   int rc;

   751   unixFile f;

   752 

   753   CRASH_TEST_OVERRIDE(sqlite3CrashOpenExclusive, zFilename, pId, delFlag);

   754   assert( 0==*pId );

   755   f.h = open(zFilename,

   756                 O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_LARGEFILE|O_BINARY,

   757                 SQLITE_DEFAULT_FILE_PERMISSIONS);

   758   if( f.h<0 ){

   759     return SQLITE_CANTOPEN;

   760   }

   761   sqlite3OsEnterMutex();

   762   rc = findLockInfo(f.h, &f.pLock, &f.pOpen);

   763   sqlite3OsLeaveMutex();

   764   if( rc ){

   765     close(f.h);

   766     unlink(zFilename);

   767     return SQLITE_NOMEM;

   768   }

   769   if( delFlag ){

   770     unlink(zFilename);

   771   }

   772   TRACE3("OPEN-EX %-3d %s\n", f.h, zFilename);

   773   return allocateUnixFile(&f, pId);

   774 }

   775 

   776 /*

   777 ** Attempt to open a new file for read-only access.

   778 **

   779 ** On success, write the file handle into *id and return SQLITE_OK.

   780 **

   781 ** On failure, return SQLITE_CANTOPEN.

   782 */

   783 int sqlite3UnixOpenReadOnly(const char *zFilename, OsFile **pId){

   784   int rc;

   785   unixFile f;

   786 

   787   CRASH_TEST_OVERRIDE(sqlite3CrashOpenReadOnly, zFilename, pId, 0);

   788   assert( 0==*pId );

   789   f.h = open(zFilename, O_RDONLY|O_LARGEFILE|O_BINARY);

   790   if( f.h<0 ){

   791     return SQLITE_CANTOPEN;

   792   }

   793   sqlite3OsEnterMutex();

   794   rc = findLockInfo(f.h, &f.pLock, &f.pOpen);

   795   sqlite3OsLeaveMutex();

   796   if( rc ){

   797     close(f.h);

   798     return SQLITE_NOMEM;

   799   }

   800   TRACE3("OPEN-RO %-3d %s\n", f.h, zFilename);

   801   return allocateUnixFile(&f, pId);

   802 }

   803 

   804 /*

   805 ** Attempt to open a file descriptor for the directory that contains a

   806 ** file.  This file descriptor can be used to fsync() the directory

   807 ** in order to make sure the creation of a new file is actually written

   808 ** to disk.

   809 **

   810 ** This routine is only meaningful for Unix.  It is a no-op under

   811 ** windows since windows does not support hard links.

   812 **

   813 ** On success, a handle for a previously open file at *id is

   814 ** updated with the new directory file descriptor and SQLITE_OK is

   815 ** returned.

   816 **

   817 ** On failure, the function returns SQLITE_CANTOPEN and leaves

   818 ** *id unchanged.

   819 */

   820 static int unixOpenDirectory(

   821   OsFile *id,

   822   const char *zDirname

   823 ){

   824   unixFile *pFile = (unixFile*)id;

   825   if( pFile==0 ){

   826     /* Do not open the directory if the corresponding file is not already

   827     ** open. */

   828     return SQLITE_CANTOPEN;

   829   }

   830   SET_THREADID(pFile);

   831   assert( pFile->dirfd<0 );

   832   pFile->dirfd = open(zDirname, O_RDONLY|O_BINARY, 0);

   833   if( pFile->dirfd<0 ){

   834     return SQLITE_CANTOPEN; 

   835   }

   836   TRACE3("OPENDIR %-3d %s\n", pFile->dirfd, zDirname);

   837   return SQLITE_OK;

   838 }

   839 

   840 /*

   841 ** If the following global variable points to a string which is the

   842 ** name of a directory, then that directory will be used to store

   843 ** temporary files.

   844 **

   845 ** See also the "PRAGMA temp_store_directory" SQL command.

   846 */

   847 char *sqlite3_temp_directory = 0;

   848 

   849 /*

   850 ** Create a temporary file name in zBuf.  zBuf must be big enough to

   851 ** hold at least SQLITE_TEMPNAME_SIZE characters.

   852 */

   853 int sqlite3UnixTempFileName(char *zBuf){

   854   static const char *azDirs[] = {

   855      0,

   856      "/var/tmp",

   857      "/usr/tmp",

   858      "/tmp",

   859      ".",

   860   };

   861   static const unsigned char zChars[] =

   862     "abcdefghijklmnopqrstuvwxyz"

   863     "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

   864     "0123456789";

   865   int i, j;

   866   struct stat buf;

   867   const char *zDir = ".";

   868   azDirs[0] = sqlite3_temp_directory;

   869   for(i=0; i<sizeof(azDirs)/sizeof(azDirs[0]); i++){

   870     if( azDirs[i]==0 ) continue;

   871     if( stat(azDirs[i], &buf) ) continue;

   872     if( !S_ISDIR(buf.st_mode) ) continue;

   873     if( access(azDirs[i], 07) ) continue;

   874     zDir = azDirs[i];

   875     break;

   876   }

   877   do{

   878     sprintf(zBuf, "%s/"TEMP_FILE_PREFIX, zDir);

   879     j = strlen(zBuf);

   880     sqlite3Randomness(15, &zBuf[j]);

   881     for(i=0; i<15; i++, j++){

   882       zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ];

   883     }

   884     zBuf[j] = 0;

   885   }while( access(zBuf,0)==0 );

   886   return SQLITE_OK; 

   887 }

   888 

   889 /*

   890 ** Check that a given pathname is a directory and is writable 

   891 **

   892 */

   893 int sqlite3UnixIsDirWritable(char *zBuf){

   894 #ifndef SQLITE_OMIT_PAGER_PRAGMAS

   895   struct stat buf;

   896   if( zBuf==0 ) return 0;

   897   if( zBuf[0]==0 ) return 0;

   898   if( stat(zBuf, &buf) ) return 0;

   899   if( !S_ISDIR(buf.st_mode) ) return 0;

   900   if( access(zBuf, 07) ) return 0;

   901 #endif /* SQLITE_OMIT_PAGER_PRAGMAS */

   902   return 1;

   903 }

   904 

   905 /*

   906 ** Seek to the offset in id->offset then read cnt bytes into pBuf.

   907 ** Return the number of bytes actually read.  Update the offset.

   908 */

   909 static int seekAndRead(unixFile *id, void *pBuf, int cnt){

   910   int got;

   911 #ifdef USE_PREAD

   912   got = pread(id->h, pBuf, cnt, id->offset);

   913 #else

   914   lseek(id->h, id->offset, SEEK_SET);

   915   got = read(id->h, pBuf, cnt);

   916 #endif

   917   if( got>0 ){

   918     id->offset += got;

   919   }

   920   return got;

   921 }

   922 

   923 /*

   924 ** Read data from a file into a buffer.  Return SQLITE_OK if all

   925 ** bytes were read successfully and SQLITE_IOERR if anything goes

   926 ** wrong.

   927 */

   928 static int unixRead(OsFile *id, void *pBuf, int amt){

   929   int got;

   930   assert( id );

   931   SimulateIOError(SQLITE_IOERR);

   932   TIMER_START;

   933   got = seekAndRead((unixFile*)id, pBuf, amt);

   934   TIMER_END;

   935   TRACE5("READ    %-3d %5d %7d %d\n", ((unixFile*)id)->h, got,

   936           last_page, TIMER_ELAPSED);

   937   SEEK(0);

   938   /* if( got<0 ) got = 0; */

   939   if( got==amt ){

   940     return SQLITE_OK;

   941   }else{

   942     return SQLITE_IOERR;

   943   }

   944 }

   945 

   946 /*

   947 ** Seek to the offset in id->offset then read cnt bytes into pBuf.

   948 ** Return the number of bytes actually read.  Update the offset.

   949 */

   950 static int seekAndWrite(unixFile *id, const void *pBuf, int cnt){

   951   int got;

   952 #ifdef USE_PREAD

   953   got = pwrite(id->h, pBuf, cnt, id->offset);

   954 #else

   955   lseek(id->h, id->offset, SEEK_SET);

   956   got = write(id->h, pBuf, cnt);

   957 #endif

   958   if( got>0 ){

   959     id->offset += got;

   960   }

   961   return got;

   962 }

   963 

   964 

   965 /*

   966 ** Write data from a buffer into a file.  Return SQLITE_OK on success

   967 ** or some other error code on failure.

   968 */

   969 static int unixWrite(OsFile *id, const void *pBuf, int amt){

   970   int wrote = 0;

   971   assert( id );

   972   assert( amt>0 );

   973   SimulateIOError(SQLITE_IOERR);

   974   SimulateDiskfullError;

   975   TIMER_START;

   976   while( amt>0 && (wrote = seekAndWrite((unixFile*)id, pBuf, amt))>0 ){

   977     amt -= wrote;

   978     pBuf = &((char*)pBuf)[wrote];

   979   }

   980   TIMER_END;

   981   TRACE5("WRITE   %-3d %5d %7d %d\n", ((unixFile*)id)->h, wrote,

   982           last_page, TIMER_ELAPSED);

   983   SEEK(0);

   984   if( amt>0 ){

   985     return SQLITE_FULL;

   986   }

   987   return SQLITE_OK;

   988 }

   989 

   990 /*

   991 ** Move the read/write pointer in a file.

   992 */

   993 static int unixSeek(OsFile *id, i64 offset){

   994   assert( id );

   995   SEEK(offset/1024 + 1);

   996 #ifdef SQLITE_TEST

   997   if( offset ) SimulateDiskfullError

   998 #endif

   999   ((unixFile*)id)->offset = offset;

  1000   return SQLITE_OK;

  1001 }

  1002 

  1003 #ifdef SQLITE_TEST

  1004 /*

  1005 ** Count the number of fullsyncs and normal syncs.  This is used to test

  1006 ** that syncs and fullsyncs are occuring at the right times.

  1007 */

  1008 int sqlite3_sync_count = 0;

  1009 int sqlite3_fullsync_count = 0;

  1010 #endif

  1011 

  1012 /*

  1013 ** Use the fdatasync() API only if the HAVE_FDATASYNC macro is defined.

  1014 ** Otherwise use fsync() in its place.

  1015 */

  1016 #ifndef HAVE_FDATASYNC

  1017 # define fdatasync fsync

  1018 #endif

  1019 

  1020 /*

  1021 ** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not

  1022 ** the F_FULLFSYNC macro is defined.  F_FULLFSYNC is currently

  1023 ** only available on Mac OS X.  But that could change.

  1024 */

  1025 #ifdef F_FULLFSYNC

  1026 # define HAVE_FULLFSYNC 1

  1027 #else

  1028 # define HAVE_FULLFSYNC 0

  1029 #endif

  1030 

  1031 

  1032 /*

  1033 ** The fsync() system call does not work as advertised on many

  1034 ** unix systems.  The following procedure is an attempt to make

  1035 ** it work better.

  1036 **

  1037 ** The SQLITE_NO_SYNC macro disables all fsync()s.  This is useful

  1038 ** for testing when we want to run through the test suite quickly.

  1039 ** You are strongly advised *not* to deploy with SQLITE_NO_SYNC

  1040 ** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash

  1041 ** or power failure will likely corrupt the database file.

  1042 */

  1043 static int full_fsync(int fd, int fullSync, int dataOnly){

  1044   int rc;

  1045 

  1046   /* Record the number of times that we do a normal fsync() and 

  1047   ** FULLSYNC.  This is used during testing to verify that this procedure

  1048   ** gets called with the correct arguments.

  1049   */

  1050 #ifdef SQLITE_TEST

  1051   if( fullSync ) sqlite3_fullsync_count++;

  1052   sqlite3_sync_count++;

  1053 #endif

  1054 

  1055   /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a

  1056   ** no-op

  1057   */

  1058 #ifdef SQLITE_NO_SYNC

  1059   rc = SQLITE_OK;

  1060 #else

  1061 

  1062 #if HAVE_FULLFSYNC

  1063   if( fullSync ){

  1064     rc = fcntl(fd, F_FULLFSYNC, 0);

  1065   }else{

  1066     rc = 1;

  1067   }

  1068   /* If the FULLSYNC failed, try to do a normal fsync() */

  1069   if( rc ) rc = fsync(fd);

  1070 

  1071 #else /* if !defined(F_FULLSYNC) */

  1072   if( dataOnly ){

  1073     rc = fdatasync(fd);

  1074   }else{

  1075     rc = fsync(fd);

  1076   }

  1077 #endif /* defined(F_FULLFSYNC) */

  1078 #endif /* defined(SQLITE_NO_SYNC) */

  1079 

  1080   return rc;

  1081 }

  1082 

  1083 /*

  1084 ** Make sure all writes to a particular file are committed to disk.

  1085 **

  1086 ** If dataOnly==0 then both the file itself and its metadata (file

  1087 ** size, access time, etc) are synced.  If dataOnly!=0 then only the

  1088 ** file data is synced.

  1089 **

  1090 ** Under Unix, also make sure that the directory entry for the file

  1091 ** has been created by fsync-ing the directory that contains the file.

  1092 ** If we do not do this and we encounter a power failure, the directory

  1093 ** entry for the journal might not exist after we reboot.  The next

  1094 ** SQLite to access the file will not know that the journal exists (because

  1095 ** the directory entry for the journal was never created) and the transaction

  1096 ** will not roll back - possibly leading to database corruption.

  1097 */

  1098 static int unixSync(OsFile *id, int dataOnly){

  1099   unixFile *pFile = (unixFile*)id;

  1100   assert( pFile );

  1101   SimulateIOError(SQLITE_IOERR);

  1102   TRACE2("SYNC    %-3d\n", pFile->h);

  1103   if( full_fsync(pFile->h, pFile->fullSync, dataOnly) ){

  1104     return SQLITE_IOERR;

  1105   }

  1106   if( pFile->dirfd>=0 ){

  1107     TRACE4("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd,

  1108             HAVE_FULLFSYNC, pFile->fullSync);

  1109 #ifndef SQLITE_DISABLE_DIRSYNC

  1110     /* The directory sync is only attempted if full_fsync is

  1111     ** turned off or unavailable.  If a full_fsync occurred above,

  1112     ** then the directory sync is superfluous.

  1113     */

  1114     if( (!HAVE_FULLFSYNC || !pFile->fullSync) && full_fsync(pFile->dirfd,0,0) ){

  1115        /*

  1116        ** We have received multiple reports of fsync() returning

  1117        ** errors when applied to directories on certain file systems.

  1118        ** A failed directory sync is not a big deal.  So it seems

  1119        ** better to ignore the error.  Ticket #1657

  1120        */

  1121        /* return SQLITE_IOERR; */

  1122     }

  1123 #endif

  1124     close(pFile->dirfd);  /* Only need to sync once, so close the directory */

  1125     pFile->dirfd = -1;    /* when we are done. */

  1126   }

  1127   return SQLITE_OK;

  1128 }

  1129 

  1130 /*

  1131 ** Sync the directory zDirname. This is a no-op on operating systems other

  1132 ** than UNIX.

  1133 **

  1134 ** This is used to make sure the master journal file has truely been deleted

  1135 ** before making changes to individual journals on a multi-database commit.