I have attached the python script I call And this is the new section in weewx.conf [[S3-Backup]] # Using S3-Backup to copy the database file to a webserver is treated as just # another report, albeit one with an unusual report generator! skin = S3Backup enable = true
# You must configure AWS at the command line and create some logon credentials # The logon credentials then go into the credentials file. # This is the name of the profile defined in that file that you use for the copy AWS_Profile = mike # This is the name of the region where the S3 bucket resides AWS_Region = eu-west-2 # This is the name of the S3 bucket where the files will be copied S3_BUCKET = s3-archive.revittmk.aws.co.uk # This determins the frequency of the backup routine report_timing = 10,30,50 * * * * # @daily On Friday, February 21, 2020 at 11:11:59 PM UTC, vince wrote: > > On Friday, February 21, 2020 at 2:52:39 PM UTC-8, Mike Revitt wrote: >> >> I have created a skin to backup my sqlite database file daily but it runs >> every archive period. Any advice on what to change to make this work >> >> >> > Can't say without seeing your skin's python code, but I'd "guess" that > it's not reading your report_timing values and acting accordingly. > > You also have "attempts" spelled incorrectly in your logging :-) > > Personally I'd just do this from cron outside weewx as a far simpler > solution, but it would be interesting to see your code. > -- You received this message because you are subscribed to the Google Groups "weewx-user" group. To unsubscribe from this group and stop receiving emails from it, send an email to weewx-user+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/weewx-user/513007f9-e955-49c5-bf39-384c1dab968e%40googlegroups.com.
# # Copyright (c) 2009-2020 Mike Revitt <m...@cougar.eu.com> # # See the file LICENSE.txt for your full rights. # """For backing up the database to S3""" import logging import os import sys import time import mimetypes import boto3 from weewx.reportengine import ReportGenerator from weeutil.weeutil import to_bool from six.moves import cPickle from datetime import datetime from botocore.exceptions import ClientError from botocore.exceptions import ParamValidationError from botocore.exceptions import ProfileNotFound log = logging.getLogger(__name__) # ============================================================================= # Class S3Backup # ============================================================================= class S3Backup(object): def __init__(self, bucket, profile, region, sqlite_root, database_name, max_tries = 3, debug = 0, secure_data = True): self.bucket = bucket self.profile = profile self.region = region self.sqlite_root = sqlite_root self.database_name = database_name self.max_tries = max_tries self.debug = debug self.secure_data = secure_data def run(self): # Add some logging information log.debug("S3Backup started at: " + datetime.now().strftime('%d-%m-%Y %H:%M:%S') + "\n") try: session = boto3.Session(profile_name=self.profile, region_name=self.region) s3 = session.resource('s3') except ProfileNotFound as e: log.error("Failed to connect to resource S3: using profile '%s'" %(self.profile)) exit(40) source_database = os.path.join(self.sqlite_root, self.database_name) dest_database = source_database.lstrip('/') # Retry up to max_tries times: for count in range(self.max_tries): try: log.debug("\nresponse=s3.meta.client.upload_file("+source_database+','+self.bucket+','+dest_database+')') response = s3.meta.client.upload_file(source_database , self.bucket, dest_database, ExtraArgs={'ContentType' : 'application/x-sqlite3'}) except IOError as e: log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tReason: %s" % (count + 1, source_database, self.bucket, dest_database, e)) except ClientError as e: log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tReason: %s" % (count + 1, source_database, self.bucket, dest_database, e)) break except ParamValidationError as e: log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tReason: %s" % (count + 1, source_database, self.bucket, dest_database, e)) break except () as e: log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tUnknown error occured $s" % (count + 1, source_database, self.bucket, dest_database, e )) break else: # Success. Log it, break out of the loop log.debug("Uploaded file %s" % source_database ) break return count # ============================================================================= # Class S3BackupGenerator # ============================================================================= class S3BackupGenerator(ReportGenerator): def run(self): import user.s3backup # determine how much logging is desired log_success = to_bool(self.skin_dict.get('log_success', True)) t1 = time.time() try: S3_backup = user.s3backup.S3Backup( bucket = self.skin_dict['S3_BUCKET'], profile = self.skin_dict['AWS_Profile'], region = self.skin_dict['AWS_Region'], sqlite_root = self.config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'], database_name = self.config_dict['Databases']['archive_sqlite']['database_name']) except KeyError as e: log.error("S3BackupGenerator: S3 Backup not requested. Skipped with error: %" % (e)) return try: n = S3_backup.run() except () as e: log.error("S3BackupGenerator: Caught exception: %s" % (e)) return if log_success: t2 = time.time() log.info("S3BackupGenerator: S3 copied files to S3 in %d attemps which took in %0.2f seconds" % (n, (t2 - t1))) # ============================================================================= # Main # ============================================================================= if __name__ == '__main__': import configobj import weewx import weeutil.logger weewx.debug = 1 weeutil.logger.setup('S3Backup', {}) if len(sys.argv) < 2: print("""Usage: S3Backup.py path-to-configuration-file [path-to-be-ftp'd]""") sys.exit(weewx.CMD_ERROR) try: config_dict = configobj.ConfigObj(sys.argv[1], file_error=True, encoding='utf-8') except IOError: print("Unable to open configuration file %s" % sys.argv[1]) raise S3_upload = S3Backup(config_dict['StdReport']['AWS-S3']['S3_BUCKET'], config_dict['StdReport']['AWS-S3']['AWS_Profile'], config_dict['StdReport']['AWS-S3']['AWS_Region'], config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'], config_dict['Databases']['archive_sqlite']['database_name']) print("\n========================================================================================\n") print("\tS3Backup started at: " + datetime.now().strftime('%d-%m-%Y %H:%M:%S') + " with the following parameters\n") print("\tBucket\t\t\tProfile\tRegion\tSQLITE Root\tDatabase Name") print("\t", config_dict['StdReport']['AWS-S3']['S3_BUCKET'], config_dict['StdReport']['AWS-S3']['AWS_Profile'], config_dict['StdReport']['AWS-S3']['AWS_Region'], config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'], config_dict['Databases']['archive_sqlite']['database_name']) print("\n========================================================================================\n") S3_upload.run()