#!/usr/bin/perl -c

# Package to handle Cyrus Index files (version 9 and version 10)

package Cyrus::IndexFile;

use strict;
use warnings;

use IO::File;
use IO::File::fcntl;
use IO::Handle;
use File::Temp;
use Data::Dumper;

# Set up header and record formatting information {{{

my $VersionFormats = {
  9 => {
    HeaderSize => 96,
    _make_fields('Header',<<EOF),
Generation            int32  4
Format                int32  4
MinorVersion          int32  4
StartOffset           int32  4
RecordSize            int32  4
Exists                int32  4
LastAppenddate        time_t 4
LastUid               int32  4
QuotaUsed             int64  8
Pop3LastLogin         time_t 4
UidValidity           int32  4
Deleted               int32  4
Answered              int32  4
Flagged               int32  4
Options               bitmap 4
LeakedCache           int32  4
HigestModseq          int64  8
Spare0                int32  4
Spare1                int32  4
Spare2                int32  4
Spare3                int32  4
Spare4                int32  4
EOF
    RecordSize => 80, # defined in file too, check it!
    _make_fields('Record', <<EOF),
Uid                   int32  4
InternalDate          time_t 4
SentDate              time_t 4
Size                  int32  4
HeaderSize            int32  4
ContentOffset         int32  4
CacheOffset           int32  4
LastUpdated           time_t 4
SystemFlags           bitmap 4
UserFlags             bitmap 16
ContentLines          int32  4
CacheVersion          int32  4
MessageUuid           hex    12
Modseq                int64  8
EOF
  },
  10 => {
    HeaderSize => 96,
    _make_fields('Header',<<EOF),
Generation            int32  4
Format                int32  4
MinorVersion          int32  4
StartOffset           int32  4
RecordSize            int32  4
Exists                int32  4
LastAppenddate        time_t 4
LastUid               int32  4
QuotaUsed             int64  8
Pop3LastLogin         time_t 4
UidValidity           int32  4
Deleted               int32  4
Answered              int32  4
Flagged               int32  4
Options               bitmap 4
LeakedCache           int32  4
HigestModseq          int64  8
Spare0                int32  4
Spare1                int32  4
Spare2                int32  4
Spare3                int32  4
Spare4                int32  4
EOF
    RecordSize => 88, # defined in file too, check it!
    _make_fields('Record', <<EOF),
Uid                   int32  4
InternalDate          time_t 4
SentDate              time_t 4
Size                  int32  4
HeaderSize            int32  4
ContentOffset         int32  4
CacheOffset           int32  4
LastUpdated           time_t 4
SystemFlags           bitmap 4
UserFlags             bitmap 16
ContentLines          int32  4
CacheVersion          int32  4
MessageGuid           hex    20
Modseq                int64  8
EOF
  },
};

sub _make_fields {
  my $prefix = shift;
  my $string = shift;

  my @lines = grep { m/\S/ } split /\n/, $string;

  my @items;
  my @packitems;
  foreach my $line (@lines) {
    my ($Name, $Type, $Size) = split /\s+/, $line;

    push @items, [$Name, $Type, $Size];
    push @packitems, _make_pack($Type, $Size);
  }

  return (
    $prefix . 'Fields' => \@items,
    $prefix . 'Pack' => join("", @packitems),
  );
}

sub _make_pack {
  my $format = shift;
  my $size = shift;
  if ($format eq 'int32' or $format eq 'time_t') {
    return 'N';
  }
  elsif ($format eq 'int64') { # ignore start..
    return 'x[N]N';
  }
  elsif ($format eq 'bitmap') {
    return 'B' . (8 * $size);
  }
  elsif ($format eq 'hex') {
    return 'H' . (2 * $size);
  }
}

# }}}

# PUBLIC API

sub new {
  my $class = shift;
  my $handle = shift;

  # read header
  my $buf;
  # XXX - check for success!
  sysread($handle, $buf, 12);
  my $version = unpack('N', substr($buf, 8));
  my $Self = bless { version => $version }, ref($class) || $class;
  if (my $frm = $VersionFormats->{$version}) {
    $Self->{format} = $frm;
    my $rest;
    sysread($handle, $rest, $frm->{HeaderSize} - 12);
    $buf .= $rest;
  }
  else {
    die "Unknown header format $version\n";
  }
  $Self->{rawheader} = $buf;
  $Self->{header} = $Self->parse_header($buf);
  $Self->{records_done} = 0;
  $Self->{handle} = $handle;
  return $Self;
}

sub new_file {
  my $class = shift;
  my $file = shift;
  my $lockopts = shift;

  my $fh;
  if ($lockopts) {
    $lockopts = ['lock_ex'] unless ref($lockopts) eq 'ARRAY';
    $fh = IO::File::fcntl->new($file, '+<', @$lockopts)
          || die "Can't open $file for locked read: $!";
  } else {
    $fh = IO::File->new("< $file") 
          || die "Can't open $file for read: $!";
  }

  return $class->new($fh);
}

sub stream_copy {
  my $Self = shift;
  my $outfh = shift;
  my $decide = shift;
  my %Opts = @_;

  my $out = $Self->new_empty($Opts{version} || $Self->{version});

  my $newheader = $Self->header_copy();

  # initially empty
  $newheader->{Exists} = 0;
  # Important!  Otherwise you get versions out of skew!
  $newheader->{MinorVersion} = $out->{version};
  $newheader->{RecordSize} = $out->{format}{RecordSize};
  $out->write_header($outfh, $newheader);

  $Self->reset();
  while (my $record = $Self->next_record()) {
    if ($decide->($newheader, $record)) {
      $newheader->{Exists}++;
      $out->write_record($outfh, $record);
    }
  }

  # update exists and last updated
  $newheader->{LastUpdated} = time();
  seek($outfh, 0, 0);
  $out->write_header($outfh, $newheader);
}

sub new_empty {
  my $class = shift;
  my $version = shift;

  my $Self = bless { version => $version }, ref($class) || $class;
  if (my $frm = $VersionFormats->{$version}) {
    $Self->{format} = $frm;
  }
  else {
    die "unknown version $version";
  }

  return $Self;
}

sub header {
  my $Self = shift;
  my $Field = shift;

  if ($Field) {
    return $Self->{header}{$Field};
  }

  return $Self->{header};
}

sub header_copy {
  my $Self = shift;

  return { %{$Self->{header}} };
}

sub reset {
  my $Self = shift;

  my $HeaderSize = $Self->{format}{HeaderSize};
  
  sysseek($Self->{handle}, $HeaderSize, 0);
  $Self->{records_done} = 0;
  delete $Self->{rawrecord};
  delete $Self->{record};
}

sub next_record {
  my $Self = shift;

  my $RecordSize = $Self->header('RecordSize');
  my $Exists = $Self->header('Exists');

  if ($Self->{records_done} < $Exists) {
    my $buf;
    sysread($Self->{handle}, $buf, $RecordSize);
    $Self->{records_done}++;
    my $rec = $Self->parse_record($buf);
    $Self->{rawrecord} = $buf;
    $Self->{record} = $rec;
    return $rec;
  }
  else {
    delete $Self->{rawrecord};
    delete $Self->{record};
    return undef; # no more records!
  }
}

sub record {
  my $Self = shift;
  my $Field = shift;

  return undef unless ($Self->{record}); 

  if ($Field) {
    return $Self->{record}{$Field};
  }
  return $Self->{record};
}

sub write_header {
  my $Self = shift;
  my $fh = shift;
  my $header = shift;

  my $buf = $Self->make_header($header);
  syswrite($fh, $buf);
}

sub write_record {
  my $Self = shift;
  my $fh = shift;
  my $record = shift;

  my $buf = $Self->make_record($record);
  syswrite($fh, $buf);
}

sub merge_indexes {
  my $Self = shift;
  my $target = shift;
  my @extras = shift;

  # copy the current header first
  my $targetpos = tell($target);
  my $header = $Self->header();
  # reset some stuff
  $header->{Exists} = 0;
  $header->{LastAppenddate} = 0;
  $header->{LastUid} = 0;
  $header->{QuotaUsed} = 0;
  $header->{Deleted} = 0;
  $header->{Answered} = 0;
  $header->{Flagged} = 0;
  $header->{HighestModseq} = 0;
  $Self->write_header($target, $header);

  my @all = ($Self, @extras);

  my @records = map { $_->next_record() } @all;

  my $nextuid = -1;

  while ($nextuid) {
    my $this;
    my $higheruid;

    # read the first record of all lists
    foreach my $n (0..$#all) {
      next unless $records[$n];
      if ($records[$n]{Uid} == $nextuid) {
        # algorithm: keep most recently modified
        if (not $this or $this->{LastModified} < $records[$n]{LastModified}) {
          $this = $records[$n]{LastModified};
        }
        # step forwards
        $records[$n] = $all[$n]->next_record();
      }
      # find the minimum now
      if (not $higheruid or $higheruid > $records[$n]{Uid}) {
        $higheruid = $records[$n]{Uid};
      }
    }

    # write out the best record found
    if ($this) {
      $Self->write_record($target, $this);
      $header->{Exists}++;
      # XXX - to make everything else work, we probably need to reconstruct or
      # put the entire logic here!
    }

    # move along
    $nextuid = $higheruid;
  }

  # move back to the start of this file and re-write the header
  seek($target, $targetpos, 0);
  $Self->write_header($target, $header);
}

sub header_dump {
  my $Self = shift;
  my $header = shift || $Self->header();
  my @data;
  foreach my $field (@{$Self->{format}{HeaderFields}}) {
    push @data, $header->{$field->[0]};
  }
  return join(' ', @data);
}

sub header_undump {
  my $Self = shift;
  my $string = shift;
  my @items = split ' ', $string;
  my %header;
  foreach my $field (@{$Self->{format}{HeaderFields}}) {
    $header{$field->[0]} = shift @items;
  }
  return \%header;
}

sub record_dump {
  my $Self = shift;
  my $record = shift || $Self->record();
  my @data;
  foreach my $field (@{$Self->{format}{RecordFields}}) {
    push @data, $record->{$field->[0]};
  }
  return join(' ', @data);
}

sub record_undump {
  my $Self = shift;
  my $string = shift;
  my @items = split ' ', $string;
  my %record;
  foreach my $field (@{$Self->{format}{RecordFields}}) {
    $record{$field->[0]} = shift @items;
  }
  return \%record;
}

# SOMEWHAT INTERNAL METHODS

sub make_header {
  my $Self = shift;
  my $ds = shift;

  my @list;
  my $fields = $Self->{format}{HeaderFields};
  foreach my $n (0..$#$fields) {
    $list[$n] = $ds->{$fields->[$n][0]};
  }

  return pack($Self->{format}{HeaderPack}, @list);
}

sub make_record {
  my $Self = shift;
  my $ds = shift;

  my @list;
  my $fields = $Self->{format}{RecordFields};
  foreach my $n (0..$#$fields) {
    $list[$n] = $ds->{$fields->[$n][0]};
  }

  return pack($Self->{format}{RecordPack}, @list);
}

sub parse_header {
  my $Self = shift;
  my $buf = shift;

  my @data = unpack($Self->{format}{HeaderPack}, $buf);
  my %res;
  my $fields = $Self->{format}{HeaderFields};
  foreach my $n (0..$#$fields) {
    $res{$fields->[$n][0]} = $data[$n];
  }

  return \%res;
}

sub parse_record {
  my $Self = shift;
  my $buf = shift;

  my @data = unpack($Self->{format}{RecordPack}, $buf);
  my %res;
  my $fields = $Self->{format}{RecordFields};
  foreach my $n (0..$#$fields) {
    $res{$fields->[$n][0]} = $data[$n];
  }

  return \%res;
}

