releasing/cbrtools/perl/MLDBM/Sync/SDBM_File.pm
author lorewang
Mon, 22 Nov 2010 10:56:31 +0800
changeset 700 c22eff170fac
parent 602 3145852acc89
permissions -rw-r--r--
update from trunk
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
602
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     1
# Copyright (c) 2001-2002 Joshua Chamas, Chamas Enterprises Inc.  All rights reserved.
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     2
# Sponsored by development on NodeWorks http://www.nodeworks.com and Apache::ASP
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     3
# http://www.apache-asp.org
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     4
#
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     5
# This program is free software; you can redistribute it
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     6
# and/or modify it under the same terms as Perl itself.
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     7
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     8
package MLDBM::Sync::SDBM_File;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
     9
$VERSION = .17;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    10
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    11
use SDBM_File;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    12
use strict;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    13
use vars qw(@ISA  $MaxSegments $MaxSegmentLength %KEYS $Zlib $VERSION);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    14
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    15
@ISA = qw(SDBM_File);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    16
$MaxSegments   = 8192; # to a 1M limit
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    17
# leave room for key index pad
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    18
$MaxSegmentLength = 128;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    19
eval "use Compress::Zlib";
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    20
$Zlib = $@ ? 0 : 1;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    21
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    22
sub FETCH {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    23
    my($self, $key) = @_;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    24
    my $segment_length = $MaxSegmentLength;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    25
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    26
    my $total_rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    27
    for(my $index = 0; $index < $MaxSegments; $index++) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    28
	my $rv = $self->SUPER::FETCH(_index_key($key, $index));
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    29
	if(defined $rv) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    30
	    $total_rv ||= '';
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    31
	    $total_rv .= $rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    32
	    last if length($rv) < $segment_length;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    33
	} else {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    34
	    last;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    35
	}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    36
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    37
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    38
    if(defined $total_rv) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    39
	$total_rv =~ s/^(..)//s;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    40
	my $type = $1;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    41
	if($type eq 'G}') {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    42
	    $total_rv = uncompress($total_rv);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    43
	} elsif ($type eq 'N}') {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    44
	    # nothing
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    45
	} else {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    46
	    # old SDBM_File ?
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    47
	    $total_rv = $type . $total_rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    48
	}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    49
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    50
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    51
    $total_rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    52
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    53
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    54
sub STORE {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    55
    my($self, $key, $value) = @_;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    56
    my $segment_length = $MaxSegmentLength;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    57
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    58
    # DELETE KEYS FIRST
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    59
    for(my $index = 0; $index < $MaxSegments; $index++) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    60
	my $index_key = _index_key($key, $index);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    61
	my $rv = $self->SUPER::FETCH($index_key);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    62
	if(defined $rv) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    63
	    $self->SUPER::DELETE($index_key);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    64
	} else {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    65
	    last;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    66
	}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    67
	last if length($rv) < $segment_length;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    68
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    69
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    70
    # G - Gzip compression
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    71
    # N - No compression
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    72
    #
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    73
    my $old_value = $value;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    74
    $value = ($Zlib && (length($value) >= $segment_length/2)) ? "G}".compress($value) : "N}".$value;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    75
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    76
    my($total_rv, $last_index);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    77
    for(my $index = 0; $index < $MaxSegments; $index++) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    78
	if($index == $MaxSegments) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    79
	    die("can't store more than $MaxSegments segments of $MaxSegmentLength bytes per key in ".__PACKAGE__);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    80
	}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    81
	$value =~ s/^(.{0,$segment_length})//so;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    82
	my $segment = $1;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    83
	
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    84
	last if length($segment) == 0;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    85
#	print "STORING "._index_key($key, $index)." $segment\n";
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    86
	my $rv = $self->SUPER::STORE(_index_key($key, $index), $segment);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    87
	$total_rv .= $segment;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    88
	$last_index = $index;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    89
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    90
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    91
#    use Time::HiRes;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    92
#    print "[".&Time::HiRes::time()."] STORED ".($last_index+1)." segments for length ".
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    93
#      length($total_rv)." bytes for value ".length($old_value)."\n";
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    94
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    95
    $old_value;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    96
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    97
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    98
sub DELETE {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
    99
    my($self, $key) = @_;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   100
    my $segment_length = $MaxSegmentLength;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   101
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   102
    my $total_rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   103
    for(my $index = 0; $index < $MaxSegments; $index++) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   104
	my $index_key = _index_key($key, $index);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   105
	my $rv = $self->SUPER::FETCH($index_key) || '';
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   106
	$self->SUPER::DELETE($index_key);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   107
	$total_rv ||= '';
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   108
	$total_rv .= $rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   109
	last if length($rv) < $segment_length;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   110
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   111
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   112
    $total_rv =~ s/^(..)//s;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   113
    my $type = $1;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   114
    if($type eq 'G}') {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   115
	$total_rv = uncompress($total_rv);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   116
    } elsif ($type eq 'N}') {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   117
	# normal
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   118
    } else {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   119
	# old SDBM_File
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   120
	$total_rv = $type.$total_rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   121
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   122
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   123
    $total_rv;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   124
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   125
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   126
sub FIRSTKEY {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   127
    my $self = shift;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   128
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   129
    my $key = $self->SUPER::FIRSTKEY();
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   130
    my @keys = ();
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   131
    if (defined $key) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   132
	do {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   133
	    if($key !~ /\*\*\d+$/s) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   134
		if(my $new_key = _decode_key($key)) {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   135
		    push(@keys, $new_key);
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   136
		}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   137
	    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   138
	} while($key = $self->SUPER::NEXTKEY($key));
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   139
    }
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   140
    $KEYS{$self} = \@keys;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   141
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   142
    $self->NEXTKEY;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   143
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   144
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   145
sub NEXTKEY {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   146
    my $self = shift;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   147
    shift(@{$KEYS{$self}});
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   148
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   149
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   150
sub _index_key {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   151
    my($key, $index) = @_;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   152
    $key =~ s/([\%\*])/uc sprintf("%%%02x",ord($1))/esg;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   153
    $index ? $key.'**'.$index : $key;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   154
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   155
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   156
sub _decode_key {
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   157
    my $key = shift;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   158
    $key =~ s/%([0-9a-fA-F]{2})/pack("c",hex($1))/ge;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   159
    $key;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   160
}
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   161
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   162
1;
3145852acc89 add releasing to new structure
jjkang
parents:
diff changeset
   163