1 #!/usr/bin/perl -w

     2 

     3 # Copyright (C) 2006, 2007 Apple Inc.  All rights reserved.

     4 #

     5 # Redistribution and use in source and binary forms, with or without

     6 # modification, are permitted provided that the following conditions

     7 # are met:

     8 #

     9 # 1.  Redistributions of source code must retain the above copyright

    10 #     notice, this list of conditions and the following disclaimer. 

    11 # 2.  Redistributions in binary form must reproduce the above copyright

    12 #     notice, this list of conditions and the following disclaimer in the

    13 #     documentation and/or other materials provided with the distribution. 

    14 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of

    15 #     its contributors may be used to endorse or promote products derived

    16 #     from this software without specific prior written permission. 

    17 #

    18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY

    19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

    20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

    21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY

    22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

    23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

    24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

    25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

    26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

    27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

    28 

    29 # This script is like the genstrings tool (minus most of the options) with these differences.

    30 #

    31 #    1) It uses the names UI_STRING and UI_STRING_WITH_KEY for the macros, rather than the macros

    32 #       from NSBundle.h, and doesn't support tables (although they would be easy to add).

    33 #    2) It supports UTF-8 in key strings (and hence uses "" strings rather than @"" strings;

    34 #       @"" strings only reliably support ASCII since they are decoded based on the system encoding

    35 #       at runtime, so give different results on US and Japanese systems for example).

    36 #    3) It looks for strings that are not marked for localization, using both macro names that are

    37 #       known to be used for debugging in Intrigue source code and an exceptions file.

    38 #    4) It finds the files to work on rather than taking them as parameters, and also uses a

    39 #       hardcoded location for both the output file and the exceptions file.

    40 #       It would have been nice to use the project to find the source files, but it's too hard to

    41 #       locate source files after parsing a .pbxproj file.

    42 

    43 # The exceptions file has a list of strings in quotes, filenames, and filename/string pairs separated by :.

    44 

    45 use strict;

    46 

    47 my $stringsFile = "English.lproj/Localizable.strings";

    48 my %isDebugMacro = ( ASSERT_WITH_MESSAGE => 1, LOG_ERROR => 1, ERROR => 1, NSURL_ERROR => 1, FATAL => 1, LOG => 1, dprintf => 1, NSException => 1, NSLog => 1, printf => 1 );

    49 

    50 @ARGV >= 1 or die "Usage: extract-localizable-strings <exceptions file> [ directory... ]\nDid you mean to run extract-webkit-localizable-strings instead?\n";

    51 

    52 my $exceptionsFile = shift @ARGV;

    53 -f $exceptionsFile or die "Couldn't find exceptions file $exceptionsFile\n";

    54 

    55 my @directories = ();

    56 my @directoriesToSkip = ();

    57 if (@ARGV < 1) {

    58     push(@directories, ".");

    59 } else {

    60     for my $dir (@ARGV) {

    61         if ($dir =~ /^-(.*)$/) {

    62             push @directoriesToSkip, $1;

    63         } else {

    64             push @directories, $dir;

    65         }

    66     }

    67 }

    68 

    69 my $sawError = 0;

    70 

    71 my $localizedCount = 0;

    72 my $keyCollisionCount = 0;

    73 my $notLocalizedCount = 0;

    74 my $NSLocalizeCount = 0;

    75 

    76 my %exception;

    77 my %usedException;

    78 

    79 if (open EXCEPTIONS, $exceptionsFile) {

    80     while (<EXCEPTIONS>) {

    81         chomp;

    82         if (/^"([^\\"]|\\.)*"$/ or /^[-_\/\w.]+.(h|m|mm|cpp)$/ or /^[-_\/\w.]+.(h|m|mm|cpp):"([^\\"]|\\.)*"$/) {

    83             if ($exception{$_}) {

    84                 print "$exceptionsFile:$.:exception for $_ appears twice\n";

    85                 print "$exceptionsFile:$exception{$_}:first appearance\n";

    86             } else {

    87                 $exception{$_} = $.;

    88             }

    89         } else {

    90             print "$exceptionsFile:$.:syntax error\n";

    91         }

    92     }

    93     close EXCEPTIONS;

    94 }

    95 

    96 my $quotedDirectoriesString = '"' . join('" "', @directories) . '"';

    97 for my $dir (@directoriesToSkip) {

    98     $quotedDirectoriesString .= ' -path "' . $dir . '" -prune';

    99 }

   100 

   101 my @files = ( split "\n", `find $quotedDirectoriesString -name "*.h" -o -name "*.m" -o -name "*.mm" -o -name "*.cpp"` );

   102 

   103 for my $file (sort @files) {

   104     next if $file =~ /\/WebLocalizableStrings\.h$/;

   105     next if $file =~ /\/icu\//;

   106 

   107     $file =~ s-^./--;

   108 

   109     open SOURCE, $file or die "can't open $file\n";

   110     

   111     my $inComment = 0;

   112     

   113     my $expected = "";

   114     my $macroLine;

   115     my $macro;

   116     my $UIString;

   117     my $key;

   118     my $comment;

   119     

   120     my $string;

   121     my $stringLine;

   122     my $nestingLevel;

   123     

   124     my $previousToken = "";

   125 

   126     while (<SOURCE>) {

   127         chomp;

   128         

   129         # Handle continued multi-line comment.

   130         if ($inComment) {

   131             next unless s-.*\*/--;

   132             $inComment = 0;

   133         }

   134     

   135         # Handle all the tokens in the line.

   136         while (s-^\s*([#\w]+|/\*|//|[^#\w/'"()\[\],]+|.)--) {

   137             my $token = $1;

   138             

   139             if ($token eq "\"") {

   140                 if ($expected and $expected ne "a quoted string") {

   141                     print "$file:$.:ERROR:found a quoted string but expected $expected\n";

   142                     $sawError = 1;

   143                     $expected = "";

   144                 }

   145                 if (s-^(([^\\$token]|\\.)*?)$token--) {

   146                     if (!defined $string) {

   147                         $stringLine = $.;

   148                         $string = $1;

   149                     } else {

   150                         $string .= $1;

   151                     }

   152                 } else {

   153                     print "$file:$.:ERROR:mismatched quotes\n";

   154                     $sawError = 1;

   155                     $_ = "";

   156                 }

   157                 next;

   158             }

   159             

   160             if (defined $string) {

   161 handleString:

   162                 if ($expected) {

   163                     if (!defined $UIString) {

   164                         # FIXME: Validate UTF-8 here?

   165                         $UIString = $string;

   166                         $expected = ",";

   167                     } elsif (($macro eq "UI_STRING_KEY" or $macro eq "LPCTSTR_UI_STRING_KEY") and !defined $key) {

   168                         # FIXME: Validate UTF-8 here?

   169                         $key = $string;

   170                         $expected = ",";

   171                     } elsif (!defined $comment) {

   172                         # FIXME: Validate UTF-8 here?

   173                         $comment = $string;

   174                         $expected = ")";

   175                     }

   176                 } else {

   177                     if (defined $nestingLevel) {

   178                         # In a debug macro, no need to localize.

   179                     } elsif ($previousToken eq "#include" or $previousToken eq "#import") {

   180                         # File name, no need to localize.

   181                     } elsif ($previousToken eq "extern" and $string eq "C") {

   182                         # extern "C", no need to localize.

   183                     } elsif ($string eq "") {

   184                         # Empty string can sometimes be localized, but we need not complain if not.

   185                     } elsif ($exception{$file}) {

   186                         $usedException{$file} = 1;

   187                     } elsif ($exception{"\"$string\""}) {

   188                         $usedException{"\"$string\""} = 1;

   189                     } elsif ($exception{"$file:\"$string\""}) {

   190                         $usedException{"$file:\"$string\""} = 1;

   191                     } else {

   192                         print "$file:$stringLine:\"$string\" is not marked for localization\n";

   193                         $notLocalizedCount++;

   194                     }

   195                 }

   196                 $string = undef;

   197                 last if !defined $token;

   198             }

   199             

   200             $previousToken = $token;

   201 

   202             if ($token =~ /^NSLocalized/ && $token !~ /NSLocalizedDescriptionKey/ && $token !~ /NSLocalizedStringFromTableInBundle/) {

   203                 print "$file:$.:ERROR:found a use of an NSLocalized macro; not supported\n";

   204                 $nestingLevel = 0 if !defined $nestingLevel;

   205                 $sawError = 1;

   206                 $NSLocalizeCount++;

   207             } elsif ($token eq "/*") {

   208                 if (!s-^.*?\*/--) {

   209                     $_ = ""; # If the comment doesn't end, discard the result of the line and set flag

   210                     $inComment = 1;

   211                 }

   212             } elsif ($token eq "//") {

   213                 $_ = ""; # Discard the rest of the line

   214             } elsif ($token eq "'") {

   215                 if (!s-([^\\]|\\.)'--) { #' <-- that single quote makes the Project Builder editor less confused

   216                     print "$file:$.:ERROR:mismatched single quote\n";

   217                     $sawError = 1;

   218                     $_ = "";

   219                 }

   220             } else {

   221                 if ($expected and $expected ne $token) {

   222                     print "$file:$.:ERROR:found $token but expected $expected\n";

   223                     $sawError = 1;

   224                     $expected = "";

   225                 }

   226                 if ($token eq "UI_STRING" or $token eq "UI_STRING_KEY" or $token eq "LPCTSTR_UI_STRING" or $token eq "LPCTSTR_UI_STRING_KEY") {

   227                     $expected = "(";

   228                     $macro = $token;

   229                     $UIString = undef;

   230                     $key = undef;

   231                     $comment = undef;

   232                     $macroLine = $.;

   233                 } elsif ($token eq "(" or $token eq "[") {

   234                     ++$nestingLevel if defined $nestingLevel;

   235                     $expected = "a quoted string" if $expected;

   236                 } elsif ($token eq ",") {

   237                     $expected = "a quoted string" if $expected;

   238                 } elsif ($token eq ")" or $token eq "]") {

   239                     $nestingLevel = undef if defined $nestingLevel && !--$nestingLevel;

   240                     if ($expected) {

   241                         $key = $UIString if !defined $key;

   242                         HandleUIString($UIString, $key, $comment, $file, $macroLine);

   243                         $macro = "";

   244                         $expected = "";

   245                         $localizedCount++;

   246                     }

   247                 } elsif ($isDebugMacro{$token}) {

   248                     $nestingLevel = 0 if !defined $nestingLevel;

   249                 }

   250             }

   251         }

   252             

   253     }

   254     

   255     goto handleString if defined $string;

   256     

   257     if ($expected) {

   258         print "$file:ERROR:reached end of file but expected $expected\n";

   259         $sawError = 1;

   260     }

   261     

   262     close SOURCE;

   263 }

   264 

   265 my %stringByKey;

   266 my %commentByKey;

   267 my %fileByKey;

   268 my %lineByKey;

   269 

   270 sub HandleUIString

   271 {

   272     my ($string, $key, $comment, $file, $line) = @_;

   273 

   274     my $bad = 0;

   275     if (grep { $_ == 0xFFFD } unpack "U*", $string) {

   276         print "$file:$line:ERROR:string for translation has illegal UTF-8 -- most likely a problem with the Text Encoding of the source file\n";

   277         $bad = 1;

   278     }

   279     if ($string ne $key && grep { $_ == 0xFFFD } unpack "U*", $key) {

   280         print "$file:$line:ERROR:key has illegal UTF-8 -- most likely a problem with the Text Encoding of the source file\n";

   281         $bad = 1;

   282     }

   283     if (grep { $_ == 0xFFFD } unpack "U*", $comment) {

   284         print "$file:$line:ERROR:comment for translation has illegal UTF-8 -- most likely a problem with the Text Encoding of the source file\n";

   285         $bad = 1;

   286     }

   287     if ($bad) {

   288         $sawError = 1;

   289         return;

   290     }

   291     

   292     if ($stringByKey{$key} && $stringByKey{$key} ne $string) {

   293         print "$file:$line:encountered the same key, \"$key\", twice, with different strings\n";

   294         print "$fileByKey{$key}:$lineByKey{$key}:previous occurrence\n";

   295         $keyCollisionCount++;

   296         return;

   297     }

   298     if ($commentByKey{$key} && $commentByKey{$key} ne $comment) {

   299         print "$file:$line:encountered the same key, \"$key\", twice, with different comments\n";

   300         print "$fileByKey{$key}:$lineByKey{$key}:previous occurrence\n";

   301         $keyCollisionCount++;

   302         return;

   303     }

   304 

   305     $fileByKey{$key} = $file;

   306     $lineByKey{$key} = $line;

   307     $stringByKey{$key} = $string;

   308     $commentByKey{$key} = $comment;

   309 }

   310 

   311 print "\n" if $sawError || $notLocalizedCount || $NSLocalizeCount;

   312 

   313 my @unusedExceptions = sort grep { !$usedException{$_} } keys %exception;

   314 if (@unusedExceptions) {

   315     for my $unused (@unusedExceptions) {

   316         print "$exceptionsFile:$exception{$unused}:exception $unused not used\n";

   317     }

   318     print "\n";

   319 }

   320 

   321 print "$localizedCount localizable strings\n" if $localizedCount;

   322 print "$keyCollisionCount key collisions\n" if $keyCollisionCount;

   323 print "$notLocalizedCount strings not marked for localization\n" if $notLocalizedCount;

   324 print "$NSLocalizeCount uses of NSLocalize\n" if $NSLocalizeCount;

   325 print scalar(@unusedExceptions), " unused exceptions\n" if @unusedExceptions;

   326 

   327 if ($sawError) {

   328     print "\nErrors encountered. Exiting without writing a $stringsFile file.\n";

   329     exit 1;

   330 }

   331 

   332 my $localizedStrings = "";

   333 

   334 for my $key (sort keys %commentByKey) {

   335     $localizedStrings .= "/* $commentByKey{$key} */\n\"$key\" = \"$stringByKey{$key}\";\n\n";

   336 }

   337 

   338 # Write out the strings file in UTF-16 with a BOM.

   339 utf8::decode($localizedStrings) if $^V ge chr(5).chr(8);

   340 my $output = pack "n*", (0xFEFF, unpack "U*", $localizedStrings);

   341 foreach my $directory (@directories) {

   342     open STRINGS, ">", "$directory/$stringsFile" or die;

   343     print STRINGS $output;

   344     close STRINGS;

   345 }