1 #!/usr/bin/env python

     2 

     3 """\

     4 List python source files.

     5 

     6 There are three functions to check whether a file is a Python source, listed

     7 here with increasing complexity:

     8 

     9 - has_python_ext() checks whether a file name ends in '.py[w]'.

    10 - look_like_python() checks whether the file is not binary and either has

    11   the '.py[w]' extension or the first line contains the word 'python'.

    12 - can_be_compiled() checks whether the file can be compiled by compile().

    13 

    14 The file also must be of appropriate size - not bigger than a megabyte.

    15 

    16 walk_python_files() recursively lists all Python files under the given directories.

    17 """

    18 __author__ = "Oleg Broytmann, Georg Brandl"

    19 

    20 __all__ = ["has_python_ext", "looks_like_python", "can_be_compiled", "walk_python_files"]

    21 

    22 

    23 import os, re

    24 

    25 binary_re = re.compile('[\x00-\x08\x0E-\x1F\x7F]')

    26 

    27 debug = False

    28 

    29 def print_debug(msg):

    30     if debug: print msg

    31 

    32 

    33 def _open(fullpath):

    34     try:

    35         size = os.stat(fullpath).st_size

    36     except OSError, err: # Permission denied - ignore the file

    37         print_debug("%s: permission denied: %s" % (fullpath, err))

    38         return None

    39 

    40     if size > 1024*1024: # too big

    41         print_debug("%s: the file is too big: %d bytes" % (fullpath, size))

    42         return None

    43 

    44     try:

    45         return open(fullpath, 'rU')

    46     except IOError, err: # Access denied, or a special file - ignore it

    47         print_debug("%s: access denied: %s" % (fullpath, err))

    48         return None

    49 

    50 def has_python_ext(fullpath):

    51     return fullpath.endswith(".py") or fullpath.endswith(".pyw")

    52 

    53 def looks_like_python(fullpath):

    54     infile = _open(fullpath)

    55     if infile is None:

    56         return False

    57 

    58     line = infile.readline()

    59     infile.close()

    60 

    61     if binary_re.search(line):

    62         # file appears to be binary

    63         print_debug("%s: appears to be binary" % fullpath)

    64         return False

    65 

    66     if fullpath.endswith(".py") or fullpath.endswith(".pyw"):

    67         return True

    68     elif "python" in line:

    69         # disguised Python script (e.g. CGI)

    70         return True

    71 

    72     return False

    73 

    74 def can_be_compiled(fullpath):

    75     infile = _open(fullpath)

    76     if infile is None:

    77         return False

    78 

    79     code = infile.read()

    80     infile.close()

    81 

    82     try:

    83         compile(code, fullpath, "exec")

    84     except Exception, err:

    85         print_debug("%s: cannot compile: %s" % (fullpath, err))

    86         return False

    87 

    88     return True

    89 

    90 

    91 def walk_python_files(paths, is_python=looks_like_python, exclude_dirs=None):

    92     """\

    93     Recursively yield all Python source files below the given paths.

    94 

    95     paths: a list of files and/or directories to be checked.

    96     is_python: a function that takes a file name and checks whether it is a

    97                Python source file

    98     exclude_dirs: a list of directory base names that should be excluded in

    99                   the search

   100     """

   101     if exclude_dirs is None:

   102         exclude_dirs=[]

   103 

   104     for path in paths:

   105         print_debug("testing: %s" % path)

   106         if os.path.isfile(path):

   107             if is_python(path):

   108                 yield path

   109         elif os.path.isdir(path):

   110             print_debug("    it is a directory")

   111             for dirpath, dirnames, filenames in os.walk(path):

   112                 for exclude in exclude_dirs:

   113                     if exclude in dirnames:

   114                         dirnames.remove(exclude)

   115                 for filename in filenames:

   116                     fullpath = os.path.join(dirpath, filename)

   117                     print_debug("testing: %s" % fullpath)

   118                     if is_python(fullpath):

   119                         yield fullpath

   120         else:

   121             print_debug("    unknown type")

   122 

   123 

   124 if __name__ == "__main__":

   125     # Two simple examples/tests

   126     for fullpath in walk_python_files(['.']):

   127         print fullpath

   128     print "----------"

   129     for fullpath in walk_python_files(['.'], is_python=can_be_compiled):

   130         print fullpath