stores, diffs and fetches CRConfig.json from the db instead of the file system
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/a4dee0d2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/a4dee0d2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/a4dee0d2 Branch: refs/heads/master Commit: a4dee0d251a9b4a7b38df9ec2439b453e833b87c Parents: 709f428 Author: Jeremy Mitchell <mitchell...@gmail.com> Authored: Fri Dec 9 12:06:14 2016 -0700 Committer: Dan Kirkwood <dang...@gmail.com> Committed: Sun Jan 8 21:05:01 2017 -0700 ---------------------------------------------------------------------- .../20161208000001_create_snapshots.sql | 29 ++++++ traffic_ops/app/lib/API/Topology.pm | 2 +- traffic_ops/app/lib/Schema/Result/Cdn.pm | 19 +++- traffic_ops/app/lib/Schema/Result/Snapshot.pm | 95 ++++++++++++++++++++ traffic_ops/app/lib/TrafficOpsRoutes.pm | 1 + traffic_ops/app/lib/UI/Snapshot.pm | 39 ++++++++ traffic_ops/app/lib/UI/Tools.pm | 2 +- traffic_ops/app/lib/UI/Topology.pm | 89 +++++++----------- traffic_ops/build/traffic_ops.spec | 3 - 9 files changed, 216 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/db/migrations/20161208000001_create_snapshots.sql ---------------------------------------------------------------------- diff --git a/traffic_ops/app/db/migrations/20161208000001_create_snapshots.sql b/traffic_ops/app/db/migrations/20161208000001_create_snapshots.sql new file mode 100644 index 0000000..240a0c5 --- /dev/null +++ b/traffic_ops/app/db/migrations/20161208000001_create_snapshots.sql @@ -0,0 +1,29 @@ +/* + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied + +-- snapshots +CREATE TABLE snapshot ( + cdn text primary key REFERENCES cdn (name) ON UPDATE CASCADE NOT NULL, + content json NOT NULL, + last_updated timestamp with time zone DEFAULT now() +); + +CREATE TRIGGER on_update_current_timestamp BEFORE UPDATE ON snapshot FOR EACH ROW EXECUTE PROCEDURE on_update_current_timestamp_last_updated(); + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/API/Topology.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/API/Topology.pm b/traffic_ops/app/lib/API/Topology.pm index fa4dcb7..6270ea8 100644 --- a/traffic_ops/app/lib/API/Topology.pm +++ b/traffic_ops/app/lib/API/Topology.pm @@ -56,7 +56,7 @@ sub SnapshotCRConfig { } my $json = &UI::Topology::gen_crconfig_json($self, $cdn_name); - &UI::Topology::write_crconfig_json($self, $cdn_name, $json); + &UI::Topology::write_crconfig_json_to_db($self, $cdn_name, $json); &UI::Utils::log($self, "Snapshot CRConfig created." , "OPER"); return $self->success("SUCCESS"); } http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/Schema/Result/Cdn.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/Schema/Result/Cdn.pm b/traffic_ops/app/lib/Schema/Result/Cdn.pm index d8c05fd..39669cb 100644 --- a/traffic_ops/app/lib/Schema/Result/Cdn.pm +++ b/traffic_ops/app/lib/Schema/Result/Cdn.pm @@ -129,9 +129,24 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 snapshot -# Created by DBIx::Class::Schema::Loader v0.07046 @ 2016-11-18 22:45:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:YT90F9iN3AECS+hDmj/hwQ +Type: might_have + +Related object: L<Schema::Result::Snapshot> + +=cut + +__PACKAGE__->might_have( + "snapshot", + "Schema::Result::Snapshot", + { "foreign.cdn" => "self.name" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-12-09 09:10:09 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:yBLkvGMimI0emk0nO5/CAA # You can replace this text with custom code or comments, and it will be preserved on regeneration http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/Schema/Result/Snapshot.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/Schema/Result/Snapshot.pm b/traffic_ops/app/lib/Schema/Result/Snapshot.pm new file mode 100644 index 0000000..08f522e --- /dev/null +++ b/traffic_ops/app/lib/Schema/Result/Snapshot.pm @@ -0,0 +1,95 @@ +use utf8; +package Schema::Result::Snapshot; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Schema::Result::Snapshot + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C<snapshot> + +=cut + +__PACKAGE__->table("snapshot"); + +=head1 ACCESSORS + +=head2 cdn + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 content + + data_type: 'json' + is_nullable: 0 + +=head2 last_updated + + data_type: 'timestamp with time zone' + default_value: current_timestamp + is_nullable: 1 + original: {default_value => \"now()"} + +=cut + +__PACKAGE__->add_columns( + "cdn", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "content", + { data_type => "json", is_nullable => 0 }, + "last_updated", + { + data_type => "timestamp with time zone", + default_value => \"current_timestamp", + is_nullable => 1, + original => { default_value => \"now()" }, + }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L</cdn> + +=back + +=cut + +__PACKAGE__->set_primary_key("cdn"); + +=head1 RELATIONS + +=head2 cdn + +Type: belongs_to + +Related object: L<Schema::Result::Cdn> + +=cut + +__PACKAGE__->belongs_to( + "cdn", + "Schema::Result::Cdn", + { name => "cdn" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-12-09 12:04:19 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:WQw7mw5c5UgACYTxspyMeg + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/TrafficOpsRoutes.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/TrafficOpsRoutes.pm b/traffic_ops/app/lib/TrafficOpsRoutes.pm index bad6232..8b7de9c 100644 --- a/traffic_ops/app/lib/TrafficOpsRoutes.pm +++ b/traffic_ops/app/lib/TrafficOpsRoutes.pm @@ -331,6 +331,7 @@ sub ui_routes { # -- Topology - CCR Config, rewrote in json $r->route('/genfiles/:mode/bycdnname/:cdnname/CRConfig')->via('GET')->over( authenticated => 1 )->to( 'Topology#ccr_config', namespace => $namespace ); + $r->get('/CRConfig-Snapshots/:cdn_name/CRConfig.json')->over( authenticated => 1 )->to( 'Snapshot#get_cdn_snapshot', namespace => $namespace ); $r->get('/types')->over( authenticated => 1 )->to( 'Types#index', namespace => $namespace ); $r->route('/types/add')->via('GET')->over( authenticated => 1 )->to( 'Types#add', namespace => $namespace ); http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/UI/Snapshot.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/UI/Snapshot.pm b/traffic_ops/app/lib/UI/Snapshot.pm new file mode 100644 index 0000000..c39a541 --- /dev/null +++ b/traffic_ops/app/lib/UI/Snapshot.pm @@ -0,0 +1,39 @@ +package UI::Snapshot; +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +# JvD Note: you always want to put Utils as the first use. Sh*t don't work if it's after the Mojo lines. +use UI::Utils; +use Mojo::Base 'Mojolicious::Controller'; +use Data::Dumper; + +sub get_cdn_snapshot { + my $self = shift; + my $cdn_name = $self->param('cdn_name'); + + my $snapshot = $self->db->resultset('Snapshot')->search( { cdn => $cdn_name } )->get_column('content')->single(); + if ( !defined($snapshot) ) { + return $self->not_found(); + } + + $self->res->headers->content_type("application/download"); + $self->res->headers->content_disposition("attachment; filename=\"CRConfig.json\""); + $self->render( text => $snapshot, format => 'json' ); + +} + +1; http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/UI/Tools.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/UI/Tools.pm b/traffic_ops/app/lib/UI/Tools.pm index 2482579..218f197 100644 --- a/traffic_ops/app/lib/UI/Tools.pm +++ b/traffic_ops/app/lib/UI/Tools.pm @@ -113,7 +113,7 @@ sub write_crconfig { $self->flash( alertmsg => $error ); } else { - UI::Topology::write_crconfig_json( $self, $cdn_name, $json ); + UI::Topology::write_crconfig_json_to_db( $self, $cdn_name, $json ); &log( $self, "Snapshot CRConfig created.", "OPER" ); $self->flash( alertmsg => "Successfully wrote CRConfig.json!" ); } http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/app/lib/UI/Topology.pm ---------------------------------------------------------------------- diff --git a/traffic_ops/app/lib/UI/Topology.pm b/traffic_ops/app/lib/UI/Topology.pm index 09fb68f..465131e 100644 --- a/traffic_ops/app/lib/UI/Topology.pm +++ b/traffic_ops/app/lib/UI/Topology.pm @@ -554,43 +554,20 @@ sub gen_crconfig_json { return ($data_obj); } -sub read_crconfig_json { - my $cdn_name = shift; - my $crconfig_file = "public/CRConfig-Snapshots/$cdn_name/CRConfig.json"; - - open my $fh, '<', $crconfig_file; - if ( $! && $! !~ m/Inappropriate ioctl for device/ ) { - my $e = Mojo::Exception->throw("$! when opening $crconfig_file"); - } - my $crconfig_disk = do { local $/; <$fh> }; - close($fh); - my $crconfig_scalar = decode_json($crconfig_disk); - return $crconfig_scalar; -} - -sub write_crconfig_json { +sub write_crconfig_json_to_db { my $self = shift; my $cdn_name = shift; my $crconfig_db = shift; my $crconfig_json = encode_json($crconfig_db); - my $crconfig_file = "public/CRConfig-Snapshots/$cdn_name/CRConfig.json"; - my $dir = dirname($crconfig_file); - - if ( !-d $dir ) { - print "$dir does not exist; attempting to create\n"; - mkpath($dir); - } - open my $fh, '>', $crconfig_file; - if ( $! && $! !~ m/Inappropriate ioctl for device/ ) { - my $e = Mojo::Exception->throw("$! when opening $crconfig_file"); + my $snapshot = $self->db->resultset('Snapshot')->find( { cdn => $cdn_name } ); + if ( defined($snapshot) ) { + $snapshot->update({ content => $crconfig_json }); + } else { + my $insert = $self->db->resultset('Snapshot')->create( { cdn => $cdn_name, content => $crconfig_json } ); + $insert->insert(); } - print $fh $crconfig_json; - close($fh); - return; - #$self->flash( alertmsg => "Success!" ); - #return $self->redirect_to($self->tx->req->content->headers->{'headers'}->{'referer'}->[0]->[0]); } sub diff_crconfig_json { @@ -598,8 +575,9 @@ sub diff_crconfig_json { my $json = shift; my $cdn_name = shift; - if ( !-f "public/CRConfig-Snapshots/$cdn_name/CRConfig.json" - && &is_admin($self) ) + my $current_snapshot = $self->db->resultset('Snapshot')->search( { cdn => $cdn_name } )->get_column('content')->single(); + + if ( !defined($current_snapshot) ) { my @err = (); $err[0] = "There is no existing CRConfig for " . $cdn_name . " to diff against... Is this the first snapshot???"; @@ -611,25 +589,24 @@ sub diff_crconfig_json { return ( \@err, \@dummy, \@caution, \@dummy, \@dummy, \@proceed, \@dummy ); } - # my $db_config = &gen_crconfig_json( $self, $cdn_name ); - my $disk_config = &read_crconfig_json($cdn_name); + $current_snapshot = decode_json($current_snapshot); ( - my $disk_ds_strings, - my $disk_loc_strings, - my $disk_cs_strings, - my $disk_csds_strings, - my $disk_rascal_strings, - my $disk_ccr_strings, - my $disk_cfg_strings - ) = &crconfig_strings($disk_config); - my @disk_ds_strings = @$disk_ds_strings; - my @disk_loc_strings = @$disk_loc_strings; - my @disk_cs_strings = @$disk_cs_strings; - my @disk_csds_strings = @$disk_csds_strings; - my @disk_rascal_strings = @$disk_rascal_strings; - my @disk_ccr_strings = @$disk_ccr_strings; - my @disk_cfg_strings = @$disk_cfg_strings; + my $ds_strings, + my $loc_strings, + my $cs_strings, + my $csds_strings, + my $rascal_strings, + my $ccr_strings, + my $cfg_strings + ) = &crconfig_strings($current_snapshot); + my @ds_strings = @$ds_strings; + my @loc_strings = @$loc_strings; + my @cs_strings = @$cs_strings; + my @csds_strings = @$csds_strings; + my @rascal_strings = @$rascal_strings; + my @ccr_strings = @$ccr_strings; + my @cfg_strings = @$cfg_strings; ( my $db_ds_strings, my $db_loc_strings, my $db_cs_strings, my $db_csds_strings, my $db_rascal_strings, my $db_ccr_strings, my $db_cfg_strings ) = &crconfig_strings($json); @@ -641,13 +618,13 @@ sub diff_crconfig_json { my @db_ccr_strings = @$db_ccr_strings; my @db_cfg_strings = @$db_cfg_strings; - my @ds_text = &compare_lists( \@db_ds_strings, \@disk_ds_strings, "Section: Delivery Services" ); - my @loc_text = &compare_lists( \@db_loc_strings, \@disk_loc_strings, "Section: Edge Cachegroups" ); - my @cs_text = &compare_lists( \@db_cs_strings, \@disk_cs_strings, "Section: Traffic Servers" ); - my @csds_text = &compare_lists( \@db_csds_strings, \@disk_csds_strings, "Section: Traffic Server - Delivery Services" ); - my @rascal_text = &compare_lists( \@db_rascal_strings, \@disk_rascal_strings, "Section: Traffic Monitors" ); - my @ccr_text = &compare_lists( \@db_ccr_strings, \@disk_ccr_strings, "Section: Traffic Routers" ); - my @cfg_text = &compare_lists( \@db_cfg_strings, \@disk_cfg_strings, "Section: CDN Configs" ); + my @ds_text = &compare_lists( \@db_ds_strings, \@ds_strings, "Section: Delivery Services" ); + my @loc_text = &compare_lists( \@db_loc_strings, \@loc_strings, "Section: Edge Cachegroups" ); + my @cs_text = &compare_lists( \@db_cs_strings, \@cs_strings, "Section: Traffic Servers" ); + my @csds_text = &compare_lists( \@db_csds_strings, \@csds_strings, "Section: Traffic Server - Delivery Services" ); + my @rascal_text = &compare_lists( \@db_rascal_strings, \@rascal_strings, "Section: Traffic Monitors" ); + my @ccr_text = &compare_lists( \@db_ccr_strings, \@ccr_strings, "Section: Traffic Routers" ); + my @cfg_text = &compare_lists( \@db_cfg_strings, \@cfg_strings, "Section: CDN Configs" ); return ( \@ds_text, \@loc_text, \@cs_text, \@csds_text, \@rascal_text, \@ccr_text, \@cfg_text ); } http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/a4dee0d2/traffic_ops/build/traffic_ops.spec ---------------------------------------------------------------------- diff --git a/traffic_ops/build/traffic_ops.spec b/traffic_ops/build/traffic_ops.spec index 253d791..b559b21 100644 --- a/traffic_ops/build/traffic_ops.spec +++ b/traffic_ops/build/traffic_ops.spec @@ -78,9 +78,6 @@ Built: %(date) by %{getenv: USER} %__cp -R $RPM_BUILD_DIR/traffic_ops-%{version}/* $RPM_BUILD_ROOT/%{PACKAGEDIR} - if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/CRConfig-Snapshots ]; then - %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/CRConfig-Snapshots - fi if [ ! -d $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/routing ]; then %__mkdir -p $RPM_BUILD_ROOT/%{PACKAGEDIR}/app/public/routing fi