Improve general robustness of console: * This covers a range of potential failures: * qpidd goes down * agent disappears from bus * agent disappear but then reappears * Add proper logging * Update console_harness to pass in logger --- console/lib/console_harness.rb | 19 ++++--- console/lib/image_builder_console.rb | 88 ++++++++++++++++++++++++++++----- 2 files changed, 85 insertions(+), 22 deletions(-)
diff --git a/console/lib/console_harness.rb b/console/lib/console_harness.rb index b7f208f..7c17384 100644 --- a/console/lib/console_harness.rb +++ b/console/lib/console_harness.rb @@ -16,26 +16,29 @@ # MA 02110-1301, USA. A copy of the GNU General Public License is # also available at http://www.gnu.org/copyleft/gpl.html. - -require 'rubygems' +$LOAD_PATH << File.expand_path(File.dirname(__FILE__)) require 'image_builder_console' require 'logger' + log = Logger.new(STDOUT) +log.level = Logger::DEBUG log.datetime_format = "%Y-%m-%d %H:%M:%S" -def get_sample +def get_sample(log) lines = [] - des = File.open("../../sample_xml/jeos_jboss.xml") + des = File.open(File.dirname(__FILE__) + "/../../sample_xml/jeos_jboss.xml") des.each {|line| lines << line } lines.join end -i = ImageBuilderConsole.new -ab = i.build_image(get_sample, "mock","some-id","security xml") +i = ImageBuilderConsole.new(:logger =>log, :retry_limit => 3, :delay => 10) +log.info "Sleeping for 10 seconds before calling build_image" +sleep 10 +ab = i.build_image(get_sample(log), "mock","some-id","security xml") log.info 'target is: ' + ab.target log.info 'status of build is: ' + ab.status loop do log.info 'checking for status update...' - log.info "Got: " + i.check_status(ab) - break if ab.status.eql?("complete") + log.info "Got: " + i.check_status(ab).to_s + break if ab.status.eql?("complete") || i.check_status(ab).to_s.eql?("failed") sleep 20 end diff --git a/console/lib/image_builder_console.rb b/console/lib/image_builder_console.rb index 6455fb7..78b7799 100644 --- a/console/lib/image_builder_console.rb +++ b/console/lib/image_builder_console.rb @@ -18,6 +18,7 @@ require 'qmf' +require 'logger' class ImageBuilderConsole < Qmf::ConsoleHandler @@ -25,8 +26,17 @@ class ImageBuilderConsole < Qmf::ConsoleHandler def initialize(args={}) @settings = Qmf::ConnectionSettings.new - args.include?(:host) ? @settings.host = args[:host] : - args.include?(:port) ? @settings.host = args[:port] : + args.include?(:host) ? @settings.host = args[:host] : nil + args.include?(:port) ? @settings.host = args[:port] : nil + @retry_limit = args.include?(:retry_limit) ? args[:retry_limit] : 20 + @delay = args.include?(:delay) ? args[:delay] : 15 + if args.include?(:logger) + @logger = args[:logger] + else + @logger = Logger.new(STDOUT) + @logger.level = Logger::DEBUG + @logger.datetime_format = "%Y-%m-%d %H:%M:%S" + end establish_connection @q = Queue.new @event_q = Queue.new @@ -42,10 +52,14 @@ class ImageBuilderConsole < Qmf::ConsoleHandler # * Returns => an ActiveBuild object # def build_image(descriptor, target,image_uuid,sec_credentials) + # TODO: pull all calls to this method out into something functionally + # equivalent to rails filters, so it runs before trying certain methods + retrieve_factory_instance unless valid_factory? @qmfc.object(Qmf::Query.new(:object_id => @ib.build_image(descriptor,target,image_uuid,sec_credentials).activebuild)) end def find_build(build_id) + @logger.info "Entering find_build method..." ab = @qmfc.objects(Qmf::Query.new(:class => "ActiveBuild")) ab.each do |build| return build if build.object_id.to_s.eql?(build_id) @@ -53,15 +67,30 @@ class ImageBuilderConsole < Qmf::ConsoleHandler end def check_status(active_build) - active_build.update - active_build.status + begin + active_build.update + active_build.status + rescue => e + @logger.error "#{e.backtrace.shift}: #{e.message}" + e.backtrace.each do |step| + @logger.error "\tfrom #{step}" + end + @logger.info "Attempting to get a new ActiveBuild object with: " + active_build.object_id.to_s + return "failed" if check_for_failed + ab = find_build(active_build.object_id) + ab.size > 0 ? check_for_failed(ab.first) : "failed" + end end def agent_added(agent) - #puts "GOT AN AGENT: #{agent} at #{Time.now.utc}" + #[email protected] "GOT AN AGENT: #{agent} at #{Time.now.utc}" #...@q << agent end + def agent_heartbeat(agent, timestamp) + #[email protected] "GOT AN AGENT HEARTBEAT: #{agent.label} at #{timestamp}" + end + #TODO: find out why this fails is enabled def event_received(event) #puts "GOT AN EVENT: #{event} at #{Time.now.utc}" @@ -75,15 +104,46 @@ class ImageBuilderConsole < Qmf::ConsoleHandler def establish_connection @connection = Qmf::Connection.new(@settings) @qmfc = Qmf::Console.new(self) - #loop do - unless @connection.connected? - #TODO: Figure out how to re-establish connection, New one with same settings? - @broker = @qmfc.add_connection(@connection) - sleep 0.5 - @broker.wait_for_stable - @ib = @qmfc.object(Qmf::Query.new(:class => "ImageBuilder")) - # sleep 15 - # end + @logger.info "trying to establish connection..." + unless @connection.connected? + @broker = @qmfc.add_connection(@connection) + sleep 0.5 + @broker.wait_for_stable + retrieve_factory_instance + end + end + + def retrieve_factory_instance + @logger.info "Trying to get an instance of ImageBuilder..." + unless @connection.connected? + establish_connection + end + attempts = 0 + loop do + attempts += 1 + @ib = @qmfc.object(Qmf::Query.new(:class => "ImageBuilder")) + #[email protected] "Retrieved: " + @ib.inspect + break if @ib + @logger.info "No ImageBuilder object found, trying again in #...@delay} seconds..." + sleep @delay + if attempts > @retry_limit + @logger.info "Giving up after #...@retry_limit} attempts" + break + end + end + return + end + + def check_for_failed + true unless @qmfc.object(Qmf::Query.new(:class => "ImageBuilder")) && @connection.connected? + end + + def valid_factory? + begin + @ib.update + return true + rescue => e + return false end end end \ No newline at end of file -- 1.7.2.3 _______________________________________________ deltacloud-devel mailing list [email protected] https://fedorahosted.org/mailman/listinfo/deltacloud-devel
