#============================================================================ 

#Name        : fileutils.py 

#Part of     : Helium 

#Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).

#All rights reserved.

#This component and the accompanying materials are made available

#under the terms of the License "Eclipse Public License v1.0"

#which accompanies this distribution, and is available

#at the URL "http://www.eclipse.org/legal/epl-v10.html".

#

#Initial Contributors:

#Nokia Corporation - initial contribution.

#

#Contributors:

#

#Description:

#===============================================================================

"""

File manipulation related functionalities:

 * Filescanner

 * rmtree (fixed version)

 * move (fixed version)

"""

import codecs

import locale

import logging

import os

import re

import sys

import shutil

import hashlib

import subprocess

import string

import pathaddition.match

import stat

if os.name == 'nt':

    import win32api

LOGGER = logging.getLogger('fileutils')

LOGGER_LOCK = logging.getLogger('fileutils.lock')

#LOGGER.addHandler(logging.FileHandler('default.log'))

#logging.basicConfig(level=logging.DEBUG)

#LOGGER.setLevel(logging.DEBUG)

class AbstractScanner(object):

    """ This class implements all the required infrastructure for filescanning. """

    def __init__(self):

        """ Initialization. """

        self.includes = []

        self.excludes = []

        self.includes_files = []

        self.excludes_files = []

        self.selectors = []

        self.filetypes = []

    def add_include(self, include):

        """ Adds an include path to the scanner. """

        if include.endswith('/') or include.endswith('\\'):

            include = include + '**'

        self.includes.append(include)

    def add_exclude(self, exclude):

        """ Adds an exclude path to the scanner. """

        if exclude.endswith('/') or exclude.endswith('\\'):

            exclude = exclude + '**'

        self.excludes.append(exclude)

    def add_exclude_file(self, exclude):

        """ Adds an exclude file to the scanner. """

        self.excludes_files.append(exclude)

    def add_selector(self, selector):

        """ Add selector to the scanner. """

        self.selectors.append(selector)

    def add_filetype(self, filetype):

        """ Adds a filetype selection to the scanner. """

        self.filetypes.append(filetype)

    def is_included(self, path):

        """ Returns if path is included by the scanner. """

        LOGGER.debug("is_included: path = " + path)

        if path.replace('\\', '/') in self.includes_files or path in self.includes_files:

            return True

        for inc in self.includes:

            if self.match(path, inc):

                LOGGER.debug("Included: " + path + " by " + inc)

                return True

        return False

    def is_excluded(self, path):

        """ Returns if path is excluded by the scanner. """

        LOGGER.debug("is_excluded: path = " + path)

        if path.replace('\\', '/') in self.excludes_files or path in self.excludes_files:

            return True

        for ex in self.excludes:

            if self.match(path, ex):

                LOGGER.debug("Excluded: " + path + " by " + ex)

                return True

        return False

    def is_selected(self, path):

        """ Returns if path is selected by all selectors in the scanner. """

        LOGGER.debug("is_selected: path = " + path)

        for selector in self.selectors:

            if not selector.is_selected(path):

                return False

        LOGGER.debug("Selected: " + path)

        return True

    def is_filetype(self, path):

        """ Test if a file matches one filetype. """

        if len(self.filetypes) == 0:

            return True

        LOGGER.debug("is_filetype: path = " + path)

        for filetype in self.filetypes:

            if self.match(path, filetype):

                LOGGER.debug("Filetype: " + path + " by " + filetype)

                return True

        return False

    def match(self, filename, pattern):

        """ Is filename matching pattern? """

        return pathaddition.match.ant_match(filename, pattern, casesensitive=(os.sep != '\\'))

    def test_path(self, root, relpath):

        """ Test if a path matches filetype, include, exclude, and selection process."""

        return self.is_filetype(relpath) and self.is_included(relpath) \

                         and not self.is_excluded(relpath) and \

                         self.is_selected(os.path.join(root, relpath))

    def __str__(self):

        """ Returns a string representing this instance. """

        content = []

        for inc in self.includes:

            content.append('include:' + os.path.normpath(inc))

        for ex in self.excludes:

            content.append('exclude:' + os.path.normpath(ex))

        return ';'.join(content)

    def __repr__(self):

        """ Returns a string representing this instance. """

        return self.__str__()

    def scan(self):

        """ Abstract method which much be overriden to implement the scanning process. """

        raise Exception("scan method must be overriden")

class FileScanner(AbstractScanner):

    """Scans the filesystem for files that match the selection paths.

    The scanner is configured with a root directory. Any number of include

    and exclude paths can be added. The scan() method is a generator that

    returns matching files one at a time when called as an iterator.

    This is a revisited implementation of the filescanner. It now relies on

    the module pathaddition.match that implements a Ant-like regular expression matcher.

    Rules:

    - Includes and excludes should not start with *

    - Includes and excludes should not have wildcard searches ending with ** (e.g. wildcard**)

    Supported includes and excludes:

    - filename.txt

    - filename.*

    - dir/

    - dir/*

    - dir/**    

    """

    def __init__(self, root_dir):

        """ Initialization. """

        AbstractScanner.__init__(self)

        self.root_dir = os.path.normpath(root_dir)

        if not self.root_dir.endswith(os.sep):

            self.root_dir = self.root_dir + os.sep

        # Add 1 so the final path separator is removed

        #self.root_dirLength = len(self.root_dir) + 1

    def scan(self):

        """ Scans the files required to zip"""

        #paths_cache = []

        excludescopy = self.excludes[:]

        for f_file in excludescopy:

            if os.path.exists(os.path.normpath(os.path.join(self.root_dir, f_file))):

                self.excludes_files.append(f_file)

                self.excludes.remove(f_file)

        includescopy = self.includes[:]

        for f_file in includescopy:

            if os.path.exists(os.path.normpath(os.path.join(self.root_dir, f_file))):

                self.includes_files.append(f_file)

                self.includes.remove(f_file)

        LOGGER.debug('Scanning sub-root directories')

        for root_dir in self.find_subroots():

            for dirpath, subdirs, files in os.walk(unicode(root_dir)):

                # Let's save the len before it's getting modified.

                subdirsLen = len(subdirs)

                subroot = dirpath[len(self.root_dir):]

                dirs_to_remove = []

                for subdir in subdirs:

                    if self.is_excluded(os.path.join(subroot, subdir)):

                        dirs_to_remove.append(subdir)

                for dir_remove in dirs_to_remove:

                    subdirs.remove(dir_remove)

                LOGGER.debug('Scanning directory: ' + dirpath)

                for file_ in files:

                    path = os.path.join(subroot, file_)

                    if self.is_filetype(path) and self.is_included(path) and \

                        self.is_selected(os.path.join(dirpath, file_)) and not self.is_excluded(path):

                        ret_path = os.path.join(dirpath, file_)

                        yield ret_path

                LOGGER.debug('Checking for empty directory: ' + dirpath)

                # Check for including empty directories

                if self.is_included(subroot) and not self.is_excluded(subroot):

                    if len(files) == 0 and subdirsLen == 0:

                        LOGGER.debug('Including empty dir: ' + dirpath)

                        yield dirpath

    def find_subroots(self):

        """Finds all the subdirectory roots based on the include paths.

        Often large archive operations define a number of archives from the root

        of the drive. Walking the tree from the root is very time-consuming, so

        selecting more specific subdirectory roots improves performance.

        """

        def splitpath(path):

            """ Returns the splitted path"""

            return path.split(os.sep)

        root_dirs = []

        # Look for includes that start with wildcards.

        subdirs_not_usable = False

        for inc in self.includes + self.includes_files:

            first_path_segment = splitpath(os.path.normpath(inc))[0]

            if first_path_segment.find('*') != -1:

                subdirs_not_usable = True

        # Parse all includes for sub-roots

        if not subdirs_not_usable:

            for inc in self.includes + self.includes_files:

                include = None

                LOGGER.debug("===> inc %s" % inc)

                contains_globs = False                

                for pathcomp in splitpath(os.path.normpath(inc)):

                    if pathcomp.find('*') != -1:

                        contains_globs = True

                        break

                    else:

                        if include == None:

                            include = pathcomp

                        else:

                            include = os.path.join(include, pathcomp)

                if not contains_globs:

                    include = os.path.dirname(include) 

                LOGGER.debug("include %s" % include)

                if include != None:

                    root_dir = os.path.normpath(os.path.join(self.root_dir, include))

                    is_new_root = True

                    for root in root_dirs[:]:

                        if destinsrc(root, root_dir):

                            LOGGER.debug("root contains include, skip it")

                            is_new_root = False

                            break

                        if destinsrc(root_dir, root):

                            LOGGER.debug("include contains root, so remove root")

                            root_dirs.remove(root)

                    if is_new_root:

                        root_dirs.append(root_dir)    

        if len(root_dirs) == 0:

            root_dirs = [os.path.normpath(self.root_dir)]

        LOGGER.debug('Roots = ' + str(root_dirs))

        return root_dirs

    def __str__(self):

        return os.path.normpath(self.root_dir) + ';' + AbstractScanner.__str__(self) 

    def __repr__(self):

        return self.__str__()

def move(src, dst):

    """Recursively move a file or directory to another location.

    If the destination is on our current filesystem, then simply use

    rename.  Otherwise, copy src to the dst and then remove src.

    A lot more could be done here...  A look at a mv.c shows a lot of

    the issues this implementation glosses over.

    """

    try:

        os.rename(src, dst)

    except OSError:

        if os.path.isdir(src):

            if destinsrc(src, dst):

                raise Exception, "Cannot move a directory '%s' into itself '%s'." % (src, dst)

            shutil.copytree(src, dst, symlinks=True)

            rmtree(src)

        else:

            shutil.copy2(src, dst)

            os.unlink(src)

def rmtree(rootdir):

    """ Catch shutil.rmtree failures on Windows when files are read-only. Thanks Google!""" 

    if sys.platform == 'win32':

        rootdir = os.path.normpath(rootdir)

        if not os.path.isabs(rootdir):

            rootdir = os.path.join(os.path.abspath('.'), rootdir)

        if not rootdir.startswith('\\\\'):

            rootdir = u"\\\\?\\" + rootdir

    def cb_handle_error(fcn, path, excinfo):

        """ Error handler, removing readonly and deleting the file. """

        os.chmod(path, 0666)

        if os.path.isdir(path):

            rmdir(path)

        elif os.path.isfile(path):

            remove(path)

        else:

            fcn(path)

    if 'java' in sys.platform:

        import java.io

        import org.apache.commons.io.FileUtils

        f_file = java.io.File(rootdir)

        org.apache.commons.io.FileUtils.deleteDirectory(f_file)

    else:

        return shutil.rmtree(rootdir, onerror=cb_handle_error)

def destinsrc(src, dst):

    """ Fixed version of destinscr, that doesn't match dst with same root name."""

    if os.sep == '\\':

        src = src.lower()

        dst = dst.lower()

    src = os.path.abspath(src)

    dst = os.path.abspath(dst)

    if not src.endswith(os.path.sep):

        src += os.path.sep

    if not dst.endswith(os.path.sep):

        dst += os.path.sep

    return dst.startswith(src)

def which(executable):

    """ Search for executable in the PATH."""

    pathlist = os.environ['PATH'].split(os.pathsep)

    pathexts = ['']

    if os.sep == '\\':

        pathexts = os.environ['PATHEXT'].split(os.pathsep)

    for folder in pathlist:

        for pathext in pathexts:

            exename = executable

            if os.sep == '\\' and not exename.lower().endswith(pathext.lower()):

                exename = exename + pathext

            filename = os.path.join(folder, exename)

            try:

                status = os.stat(filename)

            except os.error:

                continue

            # Check if the path is a regular file

            if stat.S_ISREG(status[stat.ST_MODE]):

                mode = stat.S_IMODE(status[stat.ST_MODE])

                if mode & 0111:

                    return os.path.normpath(filename)

    return None

def read_policy_content(filename):

    """ Read the policy number from the policy file.

        strict allows to activate the new policy scanning.

    """

    value = None

    error = ""

    try:

        LOGGER.debug('Opening policy file: ' + filename)

        policy_data = load_policy_content(filename)

        match = re.match(r'^((?:\d+)|(?:0842[0-9a-zA-Z]{3}))\s*$', policy_data, re.M|re.DOTALL)

        if match != None:

            value = match.group(1)

        else:

            error = "Content of '%s' doesn't match r'^\d+|0842[0-9a-zA-Z]{3}\s*$'." % filename

        error = str(exc)

    if value is not None:

        return value

    # worse case....

def load_policy_content(filename):

    """ Testing policy content loading. """

    data = ''

    try:

        fileh = codecs.open(filename, 'r', 'ascii')

        data = fileh.read()

    finally:

        fileh.close()

    return data

ENCODING_MATRIX = {

   codecs.BOM_UTF8: 'utf_8',

   codecs.BOM_UTF16: 'utf_16',

   codecs.BOM_UTF16_BE: 'utf_16_be',

   codecs.BOM_UTF16_LE: 'utf_16_le',

}

def guess_encoding(data):

    """Given a byte string, guess the encoding.

    First it tries for UTF8/UTF16 BOM.

    Next it tries the standard 'UTF8', 'ISO-8859-1', and 'cp1252' encodings,

    Plus several gathered from locale information.

    The calling program *must* first call locale.setlocale(locale.LC_ALL, '')

    If successful it returns (decoded_unicode, successful_encoding)

    If unsuccessful it raises a ``UnicodeError``.

    This was taken from http://www.voidspace.org.uk/python/articles/guessing_encoding.shtml

    """

    for bom, enc in ENCODING_MATRIX.items():

        if data.startswith(bom):

            return data.decode(enc), enc

    encodings = ['ascii', 'UTF-8']

    successful_encoding = None

    try:

        encodings.append(locale.getlocale()[1])

    except (AttributeError, IndexError):

        pass

    try:

        encodings.append(locale.getdefaultlocale()[1])

    except (AttributeError, IndexError):

        pass

    # latin-1

    encodings.append('ISO8859-1')

    encodings.append('cp1252')

    for enc in encodings:

        if not enc:

            continue

        try:

            decoded = unicode(data, enc)

            successful_encoding = enc

            break

        except (UnicodeError, LookupError):

            pass

    if successful_encoding is None:

        raise UnicodeError('Unable to decode input data. Tried the'

                           ' following encodings: %s.' %

                           ', '.join([repr(enc) for enc in encodings if enc]))

    else:

        if successful_encoding == 'ascii':

            # our default ascii encoding

            successful_encoding = 'ISO8859-1'

        return (decoded, successful_encoding)

def getmd5(fullpath, chunk_size=2**16):

    """ returns the md5 value"""

    for trial in range(3):

        try:

            file_handle = open(fullpath, "rb")

            md5 = hashlib.md5()

            while 1:

                chunk = file_handle.read(chunk_size)

                if not chunk:

                    break

                md5.update(chunk)

            file_handle.close()

            LOGGER.warning("Error happened on %d trial: %s" % (trial, str(exc)))

    return md5.hexdigest()

def read_symbian_policy_content(filename):

    """ Read the policy category from the policy file. """

    value = None

    error = ""

    try:

        LOGGER.debug('Opening symbian policy file: ' + filename)

        try:

            fileh = codecs.open(filename, 'r', 'ascii')

        for line in fileh:

            match = re.match(r'^Category\s+([A-Z])\s*$', line, re.M|re.DOTALL)

            if match != None:

                value = match.group(1)

                fileh.close()

                return value

        fileh.close()

        if match == None:

            error = "Content of '%s' doesn't match r'^Category\s+([A-Z])\s*$'." % filename

        error = str(exc)

    if value is not None:

        return value

    # worse case....

class LockFailedException(Exception):

    """ This class is used to indicate the failure in obtaining a Lock. """

    pass

if os.name == 'nt':

    import win32file