On Tue, May 23, 2017 at 09:08:44AM +0200, Dietmar Maurer wrote:
> And implement retry algorythm after failure:
>
> $next_sync = $state->{last_try} + 5*60*$fail_count;
>
> and limit to 3 failures.
>
> Signed-off-by: Dietmar Maurer <diet...@proxmox.com>
> ---
> PVE/API2/Replication.pm | 2 +-
> PVE/CLI/pvesr.pm | 28 ++++++++++++++++++++--------
> PVE/Replication.pm | 35 +++++++++++++++++++++--------------
> 3 files changed, 42 insertions(+), 23 deletions(-)
>
> diff --git a/PVE/API2/Replication.pm b/PVE/API2/Replication.pm
> index c400a93c..977d3ec1 100644
> --- a/PVE/API2/Replication.pm
> +++ b/PVE/API2/Replication.pm
> @@ -82,7 +82,7 @@ __PACKAGE__->register_method ({
> my $vmid = $d->{guest};
> next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Audit' ]);
> $d->{id} = $id;
> - foreach my $k (qw(last_sync fail_count error duration)) {
> + foreach my $k (qw(last_sync last_try fail_count error duration)) {
> $d->{$k} = $state->{$k} if defined($state->{$k});
> }
> if ($state->{pid} && $state->{ptime}) {
> diff --git a/PVE/CLI/pvesr.pm b/PVE/CLI/pvesr.pm
> index 38116f7b..115bc2c1 100644
> --- a/PVE/CLI/pvesr.pm
> +++ b/PVE/CLI/pvesr.pm
> @@ -97,14 +97,14 @@ my $print_job_list = sub {
>
> my $format = "%-20s %10s %-20s %10s %5s %8s\n";
>
> - printf($format, "JobID", "GuestID", "Target", "Interval", "Rate",
> "Enabled");
> + printf($format, "JobID", "GuestID", "Target", "Schedule", "Rate",
> "Enabled");
>
> foreach my $job (sort { $a->{guest} <=> $b->{guest} } @$list) {
> my $plugin = PVE::ReplicationConfig->lookup($job->{type});
> my $tid = $plugin->get_unique_target_id($job);
>
> printf($format, $job->{id}, $job->{guest}, $tid,
> - defined($job->{interval}) ? $job->{interval} : '-',
> + defined($job->{schedule}) ? $job->{schedule} : '*/15',
> defined($job->{rate}) ? $job->{rate} : '-',
> $job->{disable} ? 'no' : 'yes'
> );
> @@ -114,21 +114,33 @@ my $print_job_list = sub {
> my $print_job_status = sub {
> my ($list) = @_;
>
> - my $format = "%-20s %10s %-20s %20s %10s %10s %s\n";
> + my $format = "%-20s %10s %-20s %20s %20s %10s %10s %s\n";
>
> - printf($format, "JobID", "GuestID", "Target", "LastSync", "Duration",
> "FailCount", "State");
> + printf($format, "JobID", "GuestID", "Target", "LastSync", "NextSync",
> "Duration", "FailCount", "State");
>
> foreach my $job (sort { $a->{guest} <=> $b->{guest} } @$list) {
> my $plugin = PVE::ReplicationConfig->lookup($job->{type});
> my $tid = $plugin->get_unique_target_id($job);
>
> - my $timestr = $job->{last_sync} ?
> - strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{last_sync})) : '-';
> + my $timestr = '-';
> + if ($job->{last_sync}) {
> + strftime("%Y-%m-%d_%H:%M:%S", localtime($job->{last_sync}));
This is not assigned to $timestr anymore?
> + }
> +
> + my $nextstr = '-';
> + if (my $next = $job->{next_sync}) {
> + my $now = time();
> + if ($next > $now) {
> + $nextstr = strftime("%Y-%m-%d_%H:%M:%S",
> localtime($job->{next_sync}));
> + } else {
> + $nextstr = 'now'
> + }
> + }
>
> my $state = $job->{pid} ? "SYNCING" : $job->{error} // 'OK';
>
> printf($format, $job->{id}, $job->{guest}, $tid,
> - $timestr, $job->{duration} // '-',
> + $timestr, $nextstr, $job->{duration} // '-',
> $job->{fail_count}, $state);
> }
> };
> @@ -136,7 +148,7 @@ my $print_job_status = sub {
> our $cmddef = {
> status => [ 'PVE::API2::Replication', 'status', [], { node => $nodename
> }, $print_job_status ],
>
> - jobs => [ 'PVE::API2::ReplicationConfig', 'index' , [], {},
> $print_job_list ],
> + list => [ 'PVE::API2::ReplicationConfig', 'index' , [], {},
> $print_job_list ],
> read => [ 'PVE::API2::ReplicationConfig', 'read' , ['id'], {},
> sub { my $res = shift; print to_json($res, { pretty => 1,
> canonical => 1}); }],
> update => [ 'PVE::API2::ReplicationConfig', 'update' , ['id'], {} ],
> diff --git a/PVE/Replication.pm b/PVE/Replication.pm
> index ff4bbeb4..d878b44a 100644
> --- a/PVE/Replication.pm
> +++ b/PVE/Replication.pm
> @@ -9,6 +9,7 @@ use Time::HiRes qw(gettimeofday tv_interval);
> use PVE::INotify;
> use PVE::ProcFSTools;
> use PVE::Tools;
> +use PVE::CalendarEvent;
> use PVE::Cluster;
> use PVE::QemuConfig;
> use PVE::QemuServer;
> @@ -46,7 +47,8 @@ my $get_job_state = sub {
> $state = {} if !$state;
>
> $state->{last_iteration} //= 0;
> - $state->{last_sync} //= 0;
> + $state->{last_try} //= 0; # last sync start time
> + $state->{last_sync} //= 0; # last successful sync start time
> $state->{fail_count} //= 0;
>
> return $state;
> @@ -93,10 +95,23 @@ sub job_status {
>
> next if $jobcfg->{disable};
>
> - $jobcfg->{state} = $get_job_state->($stateobj, $jobcfg);
> + my $state = $get_job_state->($stateobj, $jobcfg);
> + $jobcfg->{state} = $state;
> $jobcfg->{id} = $jobid;
> $jobcfg->{vmtype} = $vms->{ids}->{$vmid}->{type};
>
> + my $next_sync = 0;
> + if (my $fail_count = $state->{fail_count}) {
> + if ($fail_count < 3) {
> + $next_sync = $state->{last_try} + 5*60*$fail_count;
> + }
> + } else {
> + my $schedule = $jobcfg->{schedule} || '*/15';
> + my $calspec = PVE::CalendarEvent::parse_calendar_event($schedule);
> + $next_sync = PVE::CalendarEvent::compute_next_event($calspec,
> $state->{last_try}) // 0;
> + }
> + $jobcfg->{next_sync} = $next_sync;
> +
> $jobs->{$jobid} = $jobcfg;
> }
>
> @@ -110,15 +125,6 @@ my $get_next_job = sub {
>
> my $jobs = job_status($stateobj);
>
> - # compute next_sync here to make it easy to sort jobs
> - my $next_sync_hash = {};
> - foreach my $jobid (keys %$jobs) {
> - my $jobcfg = $jobs->{$jobid};
> - my $interval = $jobcfg->{interval} || 15;
> - my $last_sync = $jobcfg->{state}->{last_sync};
> - $next_sync_hash->{$jobid} = $last_sync + $interval * 60;
> - }
> -
> my $sort_func = sub {
> my $joba = $jobs->{$a};
> my $jobb = $jobs->{$b};
> @@ -126,7 +132,7 @@ my $get_next_job = sub {
> my $sb = $jobb->{state};
> my $res = $sa->{last_iteration} cmp $sb->{last_iteration};
> return $res if $res != 0;
> - $res = $next_sync_hash->{$a} <=> $next_sync_hash->{$b};
> + $res = $joba->{next_sync} <=> $jobb->{next_sync};
> return $res if $res != 0;
> return $joba->{guest} <=> $jobb->{guest};
> };
> @@ -134,7 +140,7 @@ my $get_next_job = sub {
> foreach my $jobid (sort $sort_func keys %$jobs) {
> my $jobcfg = $jobs->{$jobid};
> next if $jobcfg->{state}->{last_iteration} >= $now;
> - if ($now >= $next_sync_hash->{$jobid}) {
> + if ($jobcfg->{next_sync} && ($now >= $jobcfg->{next_sync})) {
> $next_jobid = $jobid;
> last;
> }
> @@ -174,7 +180,7 @@ my $run_replication = sub {
>
> $state->{pid} = $$;
> $state->{ptime} = PVE::ProcFSTools::read_proc_starttime($state->{pid});
> -
> + $state->{last_try} = $start_time;
> $update_job_state->($stateobj, $jobcfg, $state);
>
> eval { replicate($jobcfg, $start_time); };
> @@ -188,6 +194,7 @@ my $run_replication = sub {
> $state->{fail_count}++;
> $state->{error} = "$err";
> $update_job_state->($stateobj, $jobcfg, $state);
> + warn $err;
> } else {
> $state->{last_sync} = $start_time;
> $state->{fail_count} = 0;
> --
> 2.11.0
>
> _______________________________________________
> pve-devel mailing list
> pve-devel@pve.proxmox.com
> https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
_______________________________________________
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
