diff -r ffa851df0825 -r 2fb8b9db1c86 symbian-qemu-0.9.1-12/python-2.6.1/Tools/scripts/fixdiv.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/symbian-qemu-0.9.1-12/python-2.6.1/Tools/scripts/fixdiv.py Fri Jul 31 15:01:17 2009 +0100 @@ -0,0 +1,380 @@ +#! /usr/bin/env python + +"""fixdiv - tool to fix division operators. + +To use this tool, first run `python -Qwarnall yourscript.py 2>warnings'. +This runs the script `yourscript.py' while writing warning messages +about all uses of the classic division operator to the file +`warnings'. The warnings look like this: + + :: DeprecationWarning: classic division + +The warnings are written to stderr, so you must use `2>' for the I/O +redirect. I know of no way to redirect stderr on Windows in a DOS +box, so you will have to modify the script to set sys.stderr to some +kind of log file if you want to do this on Windows. + +The warnings are not limited to the script; modules imported by the +script may also trigger warnings. In fact a useful technique is to +write a test script specifically intended to exercise all code in a +particular module or set of modules. + +Then run `python fixdiv.py warnings'. This first reads the warnings, +looking for classic division warnings, and sorts them by file name and +line number. Then, for each file that received at least one warning, +it parses the file and tries to match the warnings up to the division +operators found in the source code. If it is successful, it writes +its findings to stdout, preceded by a line of dashes and a line of the +form: + + Index: + +If the only findings found are suggestions to change a / operator into +a // operator, the output is acceptable input for the Unix 'patch' +program. + +Here are the possible messages on stdout (N stands for a line number): + +- A plain-diff-style change ('NcN', a line marked by '<', a line + containing '---', and a line marked by '>'): + + A / operator was found that should be changed to //. This is the + recommendation when only int and/or long arguments were seen. + +- 'True division / operator at line N' and a line marked by '=': + + A / operator was found that can remain unchanged. This is the + recommendation when only float and/or complex arguments were seen. + +- 'Ambiguous / operator (..., ...) at line N', line marked by '?': + + A / operator was found for which int or long as well as float or + complex arguments were seen. This is highly unlikely; if it occurs, + you may have to restructure the code to keep the classic semantics, + or maybe you don't care about the classic semantics. + +- 'No conclusive evidence on line N', line marked by '*': + + A / operator was found for which no warnings were seen. This could + be code that was never executed, or code that was only executed + with user-defined objects as arguments. You will have to + investigate further. Note that // can be overloaded separately from + /, using __floordiv__. True division can also be separately + overloaded, using __truediv__. Classic division should be the same + as either of those. (XXX should I add a warning for division on + user-defined objects, to disambiguate this case from code that was + never executed?) + +- 'Phantom ... warnings for line N', line marked by '*': + + A warning was seen for a line not containing a / operator. The most + likely cause is a warning about code executed by 'exec' or eval() + (see note below), or an indirect invocation of the / operator, for + example via the div() function in the operator module. It could + also be caused by a change to the file between the time the test + script was run to collect warnings and the time fixdiv was run. + +- 'More than one / operator in line N'; or + 'More than one / operator per statement in lines N-N': + + The scanner found more than one / operator on a single line, or in a + statement split across multiple lines. Because the warnings + framework doesn't (and can't) show the offset within the line, and + the code generator doesn't always give the correct line number for + operations in a multi-line statement, we can't be sure whether all + operators in the statement were executed. To be on the safe side, + by default a warning is issued about this case. In practice, these + cases are usually safe, and the -m option suppresses these warning. + +- 'Can't find the / operator in line N', line marked by '*': + + This really shouldn't happen. It means that the tokenize module + reported a '/' operator but the line it returns didn't contain a '/' + character at the indicated position. + +- 'Bad warning for line N: XYZ', line marked by '*': + + This really shouldn't happen. It means that a 'classic XYZ + division' warning was read with XYZ being something other than + 'int', 'long', 'float', or 'complex'. + +Notes: + +- The augmented assignment operator /= is handled the same way as the + / operator. + +- This tool never looks at the // operator; no warnings are ever + generated for use of this operator. + +- This tool never looks at the / operator when a future division + statement is in effect; no warnings are generated in this case, and + because the tool only looks at files for which at least one classic + division warning was seen, it will never look at files containing a + future division statement. + +- Warnings may be issued for code not read from a file, but executed + using an exec statement or the eval() function. These may have + in the filename position, in which case the fixdiv script + will attempt and fail to open a file named '' and issue a + warning about this failure; or these may be reported as 'Phantom' + warnings (see above). You're on your own to deal with these. You + could make all recommended changes and add a future division + statement to all affected files, and then re-run the test script; it + should not issue any warnings. If there are any, and you have a + hard time tracking down where they are generated, you can use the + -Werror option to force an error instead of a first warning, + generating a traceback. + +- The tool should be run from the same directory as that from which + the original script was run, otherwise it won't be able to open + files given by relative pathnames. +""" + +import sys +import getopt +import re +import tokenize + +multi_ok = 0 + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hm") + except getopt.error, msg: + usage(msg) + return 2 + for o, a in opts: + if o == "-h": + print __doc__ + return + if o == "-m": + global multi_ok + multi_ok = 1 + if not args: + usage("at least one file argument is required") + return 2 + if args[1:]: + sys.stderr.write("%s: extra file arguments ignored\n", sys.argv[0]) + warnings = readwarnings(args[0]) + if warnings is None: + return 1 + files = warnings.keys() + if not files: + print "No classic division warnings read from", args[0] + return + files.sort() + exit = None + for filename in files: + x = process(filename, warnings[filename]) + exit = exit or x + return exit + +def usage(msg): + sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) + sys.stderr.write("Usage: %s [-m] warnings\n" % sys.argv[0]) + sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0]) + +PATTERN = ("^(.+?):(\d+): DeprecationWarning: " + "classic (int|long|float|complex) division$") + +def readwarnings(warningsfile): + prog = re.compile(PATTERN) + try: + f = open(warningsfile) + except IOError, msg: + sys.stderr.write("can't open: %s\n" % msg) + return + warnings = {} + while 1: + line = f.readline() + if not line: + break + m = prog.match(line) + if not m: + if line.find("division") >= 0: + sys.stderr.write("Warning: ignored input " + line) + continue + filename, lineno, what = m.groups() + list = warnings.get(filename) + if list is None: + warnings[filename] = list = [] + list.append((int(lineno), intern(what))) + f.close() + return warnings + +def process(filename, list): + print "-"*70 + assert list # if this fails, readwarnings() is broken + try: + fp = open(filename) + except IOError, msg: + sys.stderr.write("can't open: %s\n" % msg) + return 1 + print "Index:", filename + f = FileContext(fp) + list.sort() + index = 0 # list[:index] has been processed, list[index:] is still to do + g = tokenize.generate_tokens(f.readline) + while 1: + startlineno, endlineno, slashes = lineinfo = scanline(g) + if startlineno is None: + break + assert startlineno <= endlineno is not None + orphans = [] + while index < len(list) and list[index][0] < startlineno: + orphans.append(list[index]) + index += 1 + if orphans: + reportphantomwarnings(orphans, f) + warnings = [] + while index < len(list) and list[index][0] <= endlineno: + warnings.append(list[index]) + index += 1 + if not slashes and not warnings: + pass + elif slashes and not warnings: + report(slashes, "No conclusive evidence") + elif warnings and not slashes: + reportphantomwarnings(warnings, f) + else: + if len(slashes) > 1: + if not multi_ok: + rows = [] + lastrow = None + for (row, col), line in slashes: + if row == lastrow: + continue + rows.append(row) + lastrow = row + assert rows + if len(rows) == 1: + print "*** More than one / operator in line", rows[0] + else: + print "*** More than one / operator per statement", + print "in lines %d-%d" % (rows[0], rows[-1]) + intlong = [] + floatcomplex = [] + bad = [] + for lineno, what in warnings: + if what in ("int", "long"): + intlong.append(what) + elif what in ("float", "complex"): + floatcomplex.append(what) + else: + bad.append(what) + lastrow = None + for (row, col), line in slashes: + if row == lastrow: + continue + lastrow = row + line = chop(line) + if line[col:col+1] != "/": + print "*** Can't find the / operator in line %d:" % row + print "*", line + continue + if bad: + print "*** Bad warning for line %d:" % row, bad + print "*", line + elif intlong and not floatcomplex: + print "%dc%d" % (row, row) + print "<", line + print "---" + print ">", line[:col] + "/" + line[col:] + elif floatcomplex and not intlong: + print "True division / operator at line %d:" % row + print "=", line + elif intlong and floatcomplex: + print "*** Ambiguous / operator (%s, %s) at line %d:" % ( + "|".join(intlong), "|".join(floatcomplex), row) + print "?", line + fp.close() + +def reportphantomwarnings(warnings, f): + blocks = [] + lastrow = None + lastblock = None + for row, what in warnings: + if row != lastrow: + lastblock = [row] + blocks.append(lastblock) + lastblock.append(what) + for block in blocks: + row = block[0] + whats = "/".join(block[1:]) + print "*** Phantom %s warnings for line %d:" % (whats, row) + f.report(row, mark="*") + +def report(slashes, message): + lastrow = None + for (row, col), line in slashes: + if row != lastrow: + print "*** %s on line %d:" % (message, row) + print "*", chop(line) + lastrow = row + +class FileContext: + def __init__(self, fp, window=5, lineno=1): + self.fp = fp + self.window = 5 + self.lineno = 1 + self.eoflookahead = 0 + self.lookahead = [] + self.buffer = [] + def fill(self): + while len(self.lookahead) < self.window and not self.eoflookahead: + line = self.fp.readline() + if not line: + self.eoflookahead = 1 + break + self.lookahead.append(line) + def readline(self): + self.fill() + if not self.lookahead: + return "" + line = self.lookahead.pop(0) + self.buffer.append(line) + self.lineno += 1 + return line + def truncate(self): + del self.buffer[-window:] + def __getitem__(self, index): + self.fill() + bufstart = self.lineno - len(self.buffer) + lookend = self.lineno + len(self.lookahead) + if bufstart <= index < self.lineno: + return self.buffer[index - bufstart] + if self.lineno <= index < lookend: + return self.lookahead[index - self.lineno] + raise KeyError + def report(self, first, last=None, mark="*"): + if last is None: + last = first + for i in range(first, last+1): + try: + line = self[first] + except KeyError: + line = "" + print mark, chop(line) + +def scanline(g): + slashes = [] + startlineno = None + endlineno = None + for type, token, start, end, line in g: + endlineno = end[0] + if startlineno is None: + startlineno = endlineno + if token in ("/", "/="): + slashes.append((start, line)) + if type == tokenize.NEWLINE: + break + return startlineno, endlineno, slashes + +def chop(line): + if line.endswith("\n"): + return line[:-1] + else: + return line + +if __name__ == "__main__": + sys.exit(main())