Changes have been pushed for the repository "fawkesrobotics/fawkes".
Clone: https://github.com/fawkesrobotics/fawkes.git Gitweb: https://github.com/fawkesrobotics/fawkes The branch, tviehmann/lookup-estimator-script-improvements has been updated to 9e0dcd646a8a0dc286c2b21466f8fcaf683c4e3c (commit) via cb2d9a934052dbb215d7ad0739dbe57a0595746c (commit) from 9df76dec7e726b26ab50fa052023ff86329ab555 (commit) https://github.com/fawkesrobotics/fawkes/tree/tviehmann/lookup-estimator-script-improvements Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - *Log* --------------------------------------------------------------- commit cb2d9a934052dbb215d7ad0739dbe57a0595746c Author: Tarik Viehmann <[email protected]> AuthorDate: Fri Mar 19 13:58:28 2021 +0100 Commit: Tarik Viehmann <[email protected]> CommitDate: Wed May 5 14:02:07 2021 +0200 scripts: add docstring to all functions and classes https://github.com/fawkesrobotics/fawkes/commit/cb2d9a934 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - commit 9e0dcd646a8a0dc286c2b21466f8fcaf683c4e3c Author: Tarik Viehmann <[email protected]> AuthorDate: Fri Mar 19 14:10:49 2021 +0100 Commit: Tarik Viehmann <[email protected]> CommitDate: Wed May 5 14:02:57 2021 +0200 scripts: reformat lookup script with black https://github.com/fawkesrobotics/fawkes/commit/9e0dcd646 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - *Summary* ----------------------------------------------------------- etc/scripts/mongodb_skillsim_lookup.py | 664 +++++++++++++++++++-------------- 1 file changed, 388 insertions(+), 276 deletions(-) - *Diffs* ------------------------------------------------------------- - *commit* cb2d9a934052dbb215d7ad0739dbe57a0595746c - - - - - - - - - - Author: Tarik Viehmann <[email protected]> Date: Fri Mar 19 13:58:28 2021 +0100 Subject: scripts: add docstring to all functions and classes etc/scripts/mongodb_skillsim_lookup.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) _Diff for modified files_: diff --git a/etc/scripts/mongodb_skillsim_lookup.py b/etc/scripts/mongodb_skillsim_lookup.py index db049b544..973d470f4 100755 --- a/etc/scripts/mongodb_skillsim_lookup.py +++ b/etc/scripts/mongodb_skillsim_lookup.py @@ -43,9 +43,11 @@ S_FINAL = 1 S_FAILED = 3 def time_diff_in_sec(end, start): + """ Computee the time diff in seconds from ms precision times. """ return int(max((end - start)/1000, 0)) def split_skill_string(skill): + """ Separate skill name and arguments from a fawkes skill string. """ skill = skill.strip().replace(" ","").replace("\n","") skill_regex = "([\w_-]+)\{(.*)\}" try_match = re.match(skill_regex, skill) @@ -62,10 +64,12 @@ def split_skill_string(skill): raise Exception("invalid skill format", skill) def drop_collection(mongodb_uri, database, collection): + """ Drop a collection from a mongodb instance. """ curr_client = pymongo.MongoClient(mongodb_uri) curr_client[database][collection].drop() class GaussSampler: +""" Sampler to obtain points from a meixture of gaussians. """ def __init__( self, @@ -74,6 +78,7 @@ class GaussSampler: gauss_params, upper_bound, lower_bound): + """ Sample points from a bounded and weighted mixture of gaussians. """ self.dist_weights = dist_weights self.lower_bound = lower_bound self.upper_bound = upper_bound @@ -114,6 +119,7 @@ class GaussSampler: break def display(self, bin_size): + """ Show the sampled points using pyplot. """ xs = np.linspace(self.sample_min, self.sample_max, 2000) ys = np.zeros_like(xs) for (l, s), w in zip(self.gauss_params, self.dist_weights): @@ -131,6 +137,7 @@ class GaussSampler: class MongoInterface: +""" Interface to a mongodb instance. """ def __init__( self, @@ -138,12 +145,14 @@ class MongoInterface: dst_database, dst_collection, dry_run): + """ Set up a connection to a mongodb collection. """ self.client = pymongo.MongoClient(dst_mongodb_uri) self.dst_mongodb_uri = dst_mongodb_uri self.lookup_col = self.client[dst_database][dst_collection] self.dry_run = dry_run def upload(self, durations, skill_name, skill_args): + """ Upload skill exec time entries, randomly sample from skill arg choices. """ for dur in durations: doc = {"outcome": 1, "error": "", "name": skill_name, "duration": dur} args_dict = dict() @@ -159,6 +168,7 @@ class MongoInterface: self.lookup_col.insert_one(doc) def transform(self, src_mongodb_uri, src_database, src_collection, lower_bound, upper_bound): + """ Transform blackboard log data from the Skiller to exec time entries. """ if src_mongodb_uri != self.dst_mongodb_uri: self.clone_collection(src_mongodb_uri, src_database, src_collection) col = self.client[src_database][src_collection] @@ -190,6 +200,7 @@ class MongoInterface: self.client.drop_database(src_database) def clone_collection(self, src_mongodb_uri, src_database,src_collection): + """ Clone a mongodb collection to the client. """ # drop "mongodb://" suffix from uri src_conn = src_mongodb_uri[10:] if src_conn[-1] == "/": @@ -199,6 +210,8 @@ class MongoInterface: def main(): header = ''' + """ Obtain data for the lookup execution time estimator. """ + header = """ ############################################################################### # # # Obtain data for the lookup execution time estimator # @@ -377,5 +390,11 @@ example call: ./mongodb_skillsim_lookup.py generate -d -n \ print("unrecognized mode") +<<<<<<< HEAD if __name__ == '__main__': main() +======= +if __name__ == "__main__": + """ Invoke the main function. """ + main() +>>>>>>> d55b351fd (fixup! scripts: add docstring to all functions and classes) - *commit* 9e0dcd646a8a0dc286c2b21466f8fcaf683c4e3c - - - - - - - - - - Author: Tarik Viehmann <[email protected]> Date: Fri Mar 19 14:10:49 2021 +0100 Subject: scripts: reformat lookup script with black etc/scripts/mongodb_skillsim_lookup.py | 667 +++++++++++++++++++-------------- 1 file changed, 380 insertions(+), 287 deletions(-) _Diff for modified files_: diff --git a/etc/scripts/mongodb_skillsim_lookup.py b/etc/scripts/mongodb_skillsim_lookup.py index 973d470f4..1b7720295 100755 --- a/etc/scripts/mongodb_skillsim_lookup.py +++ b/etc/scripts/mongodb_skillsim_lookup.py @@ -42,20 +42,22 @@ S_RUNNING = 2 S_FINAL = 1 S_FAILED = 3 + def time_diff_in_sec(end, start): - """ Computee the time diff in seconds from ms precision times. """ - return int(max((end - start)/1000, 0)) + """ Computee the time diff in seconds from ms precision times. """ + return int(max((end - start) / 1000, 0)) + def split_skill_string(skill): - """ Separate skill name and arguments from a fawkes skill string. """ - skill = skill.strip().replace(" ","").replace("\n","") + """ Separate skill name and arguments from a fawkes skill string. """ + skill = skill.strip().replace(" ", "").replace("\n", "") skill_regex = "([\w_-]+)\{(.*)\}" try_match = re.match(skill_regex, skill) if try_match: - name, args_str = try_match.group(1,2) + name, args_str = try_match.group(1, 2) arg_regex = '([\w_-]+)=\\"([\w_-]+)\\"(?:,)?' # split along the regex, filtering empty matches - arg_list = list(filter(None,re.split(arg_regex, args_str))) + arg_list = list(filter(None, re.split(arg_regex, args_str))) # convert list to dictionary with even odd matching # see https://stackoverflow.com/a/23286311 args = dict(zip(arg_list[::2], arg_list[1::2])) @@ -63,153 +65,180 @@ def split_skill_string(skill): else: raise Exception("invalid skill format", skill) + def drop_collection(mongodb_uri, database, collection): - """ Drop a collection from a mongodb instance. """ - curr_client = pymongo.MongoClient(mongodb_uri) - curr_client[database][collection].drop() + """ Drop a collection from a mongodb instance. """ + curr_client = pymongo.MongoClient(mongodb_uri) + curr_client[database][collection].drop() + class GaussSampler: -""" Sampler to obtain points from a meixture of gaussians. """ + """ Sampler to obtain points from a meixture of gaussians. """ - def __init__( - self, - quantity, - dist_weights, - gauss_params, - upper_bound, - lower_bound): - """ Sample points from a bounded and weighted mixture of gaussians. """ - self.dist_weights = dist_weights - self.lower_bound = lower_bound - self.upper_bound = upper_bound - if len(self.dist_weights) != len(gauss_params): - print("Number of distribution weights do not match number of distributions!") - diff= len(gauss_params) - len(dist_weights) - if diff < 0: - print("Ignoring trailing distribution weights") - self.dist_weights = self.dist_weights[:len(dist_weights)+diff] - else: - print("Assuming default weights of 1") - self.dist_weights.extend([1]*diff) - # normalize weights - self.dist_weights = np.array([float(i)/sum(self.dist_weights) for i in self.dist_weights]) - # create samples - self.samples = [] - self.gauss_params=gauss_params - sample_size = quantity - self.sample_min, self.sample_max = [float("inf"),-float("inf")] - while(True): - # determine the gaussian to sample from for each sample - mixture_idx = np.random.choice(len(self.dist_weights), - size=sample_size, - replace=True, - p=self.dist_weights) - # create the samples from the respective gaussian - temp = np.fromiter((ss.norm.rvs(*(gauss_params[i])) - for i in mixture_idx), dtype=np.float64) - # remember mixed sampled extremas for plotting - self.sample_min = min(self.sample_min,temp.min()) - self.sample_max = max(self.sample_max,temp.max()) - # add those samples that are within the bounds - self.samples = np.concatenate([self.samples,np.fromiter([x for x in temp - if x <= upper_bound and x >= lower_bound], - dtype=np.float64)]) - sample_size = quantity-len(self.samples) - if(sample_size == 0): - break + def __init__(self, quantity, dist_weights, gauss_params, upper_bound, lower_bound): + """ Sample points from a bounded and weighted mixture of gaussians. """ + self.dist_weights = dist_weights + self.lower_bound = lower_bound + self.upper_bound = upper_bound + if len(self.dist_weights) != len(gauss_params): + print( + "Number of distribution weights do not match number of distributions!" + ) + diff = len(gauss_params) - len(dist_weights) + if diff < 0: + print("Ignoring trailing distribution weights") + self.dist_weights = self.dist_weights[: len(dist_weights) + diff] + else: + print("Assuming default weights of 1") + self.dist_weights.extend([1] * diff) + # normalize weights + self.dist_weights = np.array( + [float(i) / sum(self.dist_weights) for i in self.dist_weights] + ) + # create samples + self.samples = [] + self.gauss_params = gauss_params + sample_size = quantity + self.sample_min, self.sample_max = [float("inf"), -float("inf")] + while True: + # determine the gaussian to sample from for each sample + mixture_idx = np.random.choice( + len(self.dist_weights), + size=sample_size, + replace=True, + p=self.dist_weights, + ) + # create the samples from the respective gaussian + temp = np.fromiter( + (ss.norm.rvs(*(gauss_params[i])) for i in mixture_idx), dtype=np.float64 + ) + # remember mixed sampled extremas for plotting + self.sample_min = min(self.sample_min, temp.min()) + self.sample_max = max(self.sample_max, temp.max()) + # add those samples that are within the bounds + self.samples = np.concatenate( + [ + self.samples, + np.fromiter( + [x for x in temp if x <= upper_bound and x >= lower_bound], + dtype=np.float64, + ), + ] + ) + sample_size = quantity - len(self.samples) + if sample_size == 0: + break - def display(self, bin_size): - """ Show the sampled points using pyplot. """ - xs = np.linspace(self.sample_min, self.sample_max, 2000) - ys = np.zeros_like(xs) - for (l, s), w in zip(self.gauss_params, self.dist_weights): - ys += ss.norm.pdf(xs, loc=l, scale=s) * w - plt.plot(xs, ys,color="blue") - plt.hist(self.samples, density=True, bins=bin_size,color='palegreen') - plt.xlabel("duration") - plt.ylabel("density") - _, _, ymin, ymax = plt.axis() - if(self.lower_bound > 0): - plt.vlines([self.lower_bound],ymin,ymax,color='crimson') - if(self.upper_bound < float('inf')): - plt.vlines([self.upper_bound],ymin,ymax,color='crimson') - plt.show() + def display(self, bin_size): + """ Show the sampled points using pyplot. """ + xs = np.linspace(self.sample_min, self.sample_max, 2000) + ys = np.zeros_like(xs) + for (l, s), w in zip(self.gauss_params, self.dist_weights): + ys += ss.norm.pdf(xs, loc=l, scale=s) * w + plt.plot(xs, ys, color="blue") + plt.hist(self.samples, density=True, bins=bin_size, color="palegreen") + plt.xlabel("duration") + plt.ylabel("density") + _, _, ymin, ymax = plt.axis() + if self.lower_bound > 0: + plt.vlines([self.lower_bound], ymin, ymax, color="crimson") + if self.upper_bound < float("inf"): + plt.vlines([self.upper_bound], ymin, ymax, color="crimson") + plt.show() class MongoInterface: -""" Interface to a mongodb instance. """ + """ Interface to a mongodb instance. """ - def __init__( - self, - dst_mongodb_uri, - dst_database, - dst_collection, - dry_run): - """ Set up a connection to a mongodb collection. """ - self.client = pymongo.MongoClient(dst_mongodb_uri) - self.dst_mongodb_uri = dst_mongodb_uri - self.lookup_col = self.client[dst_database][dst_collection] - self.dry_run = dry_run + def __init__(self, dst_mongodb_uri, dst_database, dst_collection, dry_run): + """ Set up a connection to a mongodb collection. """ + self.client = pymongo.MongoClient(dst_mongodb_uri) + self.dst_mongodb_uri = dst_mongodb_uri + self.lookup_col = self.client[dst_database][dst_collection] + self.dry_run = dry_run - def upload(self, durations, skill_name, skill_args): - """ Upload skill exec time entries, randomly sample from skill arg choices. """ - for dur in durations: - doc = {"outcome": 1, "error": "", "name": skill_name, "duration": dur} - args_dict = dict() - if skill_args != None: - for arg in skill_args: - if(len(arg) == 1): - args_dict[arg[0]]='.*' - else: - args_dict[arg[0]]=np.random.choice(arg[1:]) - doc["args"] = args_dict - print(doc) - if(not self.dry_run): - self.lookup_col.insert_one(doc) + def upload(self, durations, skill_name, skill_args): + """ Upload skill exec time entries, randomly sample from skill arg choices. """ + for dur in durations: + doc = {"outcome": 1, "error": "", "name": skill_name, "duration": dur} + args_dict = dict() + if skill_args != None: + for arg in skill_args: + if len(arg) == 1: + args_dict[arg[0]] = ".*" + else: + args_dict[arg[0]] = np.random.choice(arg[1:]) + doc["args"] = args_dict + print(doc) + if not self.dry_run: + self.lookup_col.insert_one(doc) - def transform(self, src_mongodb_uri, src_database, src_collection, lower_bound, upper_bound): - """ Transform blackboard log data from the Skiller to exec time entries. """ - if src_mongodb_uri != self.dst_mongodb_uri: - self.clone_collection(src_mongodb_uri, src_database, src_collection) - col = self.client[src_database][src_collection] - for skill_start in col.find({"status": S_RUNNING}).sort("timestamp", 1): - for skill_end in col.find({"skill_string": skill_start["skill_string"], - "thread": skill_start["thread"], - "timestamp": {"$gt": skill_start["timestamp"]} - }).sort("timestamp", 1).limit(1): - if skill_end["status"] == S_FINAL or skill_end["status"] == S_FAILED: - name, args = split_skill_string(skill_start["skill_string"]) - lookup_entry = {"_id": {"thread": skill_start["thread"], - "start_time": skill_start["timestamp"], - "end_time": skill_end["timestamp"]}, - "outcome": skill_end["status"], - "error": skill_end["error"], - "name": name, - "args": args, - "duration": time_diff_in_sec(skill_end["timestamp"],skill_start["timestamp"])} - if lookup_entry["duration"] > upper_bound or lookup_entry["duration"] < lower_bound: - print("duration out of bounds, omitting: {} seconds\n{}\n{}".format(lookup_entry["duration"], skill_start, skill_end)) - else: - if not self.lookup_col.find_one(lookup_entry): - if(not self.dry_run): - self.lookup_col.insert_one(lookup_entry) - print("Adding: {}".format(lookup_entry)) - else: - print("Entry already present, omitting") - if src_mongodb_uri != self.dst_mongodb_uri: - self.client.drop_database(src_database) + def transform( + self, src_mongodb_uri, src_database, src_collection, lower_bound, upper_bound + ): + """ Transform blackboard log data from the Skiller to exec time entries. """ + if src_mongodb_uri != self.dst_mongodb_uri: + self.clone_collection(src_mongodb_uri, src_database, src_collection) + col = self.client[src_database][src_collection] + for skill_start in col.find({"status": S_RUNNING}).sort("timestamp", 1): + for skill_end in ( + col.find( + { + "skill_string": skill_start["skill_string"], + "thread": skill_start["thread"], + "timestamp": {"$gt": skill_start["timestamp"]}, + } + ) + .sort("timestamp", 1) + .limit(1) + ): + if skill_end["status"] == S_FINAL or skill_end["status"] == S_FAILED: + name, args = split_skill_string(skill_start["skill_string"]) + lookup_entry = { + "_id": { + "thread": skill_start["thread"], + "start_time": skill_start["timestamp"], + "end_time": skill_end["timestamp"], + }, + "outcome": skill_end["status"], + "error": skill_end["error"], + "name": name, + "args": args, + "duration": time_diff_in_sec( + skill_end["timestamp"], skill_start["timestamp"] + ), + } + if ( + lookup_entry["duration"] > upper_bound + or lookup_entry["duration"] < lower_bound + ): + print( + "duration out of bounds, omitting: {} seconds\n{}\n{}".format( + lookup_entry["duration"], skill_start, skill_end + ) + ) + else: + if not self.lookup_col.find_one(lookup_entry): + if not self.dry_run: + self.lookup_col.insert_one(lookup_entry) + print("Adding: {}".format(lookup_entry)) + else: + print("Entry already present, omitting") + if src_mongodb_uri != self.dst_mongodb_uri: + self.client.drop_database(src_database) - def clone_collection(self, src_mongodb_uri, src_database,src_collection): - """ Clone a mongodb collection to the client. """ - # drop "mongodb://" suffix from uri - src_conn = src_mongodb_uri[10:] - if src_conn[-1] == "/": - src_conn = src_conn[:-1] - self.client.admin.command({'cloneCollection': src_database+"."+src_collection, 'from': src_conn}) + def clone_collection(self, src_mongodb_uri, src_database, src_collection): + """ Clone a mongodb collection to the client. """ + # drop "mongodb://" suffix from uri + src_conn = src_mongodb_uri[10:] + if src_conn[-1] == "/": + src_conn = src_conn[:-1] + self.client.admin.command( + {"cloneCollection": src_database + "." + src_collection, "from": src_conn} + ) def main(): - header = ''' """ Obtain data for the lookup execution time estimator. """ header = """ ############################################################################### @@ -223,174 +252,234 @@ def main(): # 2. samples of a mixed gaussian distribution # # # ############################################################################### -''' - parser = argparse.ArgumentParser(description=textwrap.dedent(header), - formatter_class=argparse.RawTextHelpFormatter) - common = argparse.ArgumentParser(add_help=False) - group = common.add_argument_group("Mongodb options") - group.add_argument( - '--mongodb-uri', - type=str, - help='The MongoDB URI of the execution time estimator lookup database (default: %(default)s)', - default='mongodb://localhost:27017/') - group.add_argument( - '--db', - type=str, - help=textwrap.dedent('''name of the lookup database (default: %(default)s)'''), - default='skills') - group.add_argument( - '--dry-run', '-d', - action='store_true', - help='only create samples without uploading them to mongodb') - group.add_argument( - '--collection', '-c', - type=str, - help='name of the lookup collection (default: %(default)s)', - default='exec_times') - group.add_argument( - '--drop-collection-first', '-dc', - action='store_true', - help='clear all old data from the collection') - subparsers = parser.add_subparsers(help='Source of the execution time data',dest='subparser') - bb_parser = subparsers.add_parser("bblog",parents=[common], - description=textwrap.dedent(header + '''\ +""" + parser = argparse.ArgumentParser( + description=textwrap.dedent(header), + formatter_class=argparse.RawTextHelpFormatter, + ) + common = argparse.ArgumentParser(add_help=False) + group = common.add_argument_group("Mongodb options") + group.add_argument( + "--mongodb-uri", + type=str, + help="The MongoDB URI of the execution time estimator lookup database (default: %(default)s)", + default="mongodb://localhost:27017/", + ) + group.add_argument( + "--db", + type=str, + help=textwrap.dedent("""name of the lookup database (default: %(default)s)"""), + default="skills", + ) + group.add_argument( + "--dry-run", + "-d", + action="store_true", + help="only create samples without uploading them to mongodb", + ) + group.add_argument( + "--collection", + "-c", + type=str, + help="name of the lookup collection (default: %(default)s)", + default="exec_times", + ) + group.add_argument( + "--drop-collection-first", + "-dc", + action="store_true", + help="clear all old data from the collection", + ) + subparsers = parser.add_subparsers( + help="Source of the execution time data", dest="subparser" + ) + bb_parser = subparsers.add_parser( + "bblog", + parents=[common], + description=textwrap.dedent( + header + + """\ # # # Selected option 1 # # # ############################################################################### -'''),formatter_class=argparse.RawTextHelpFormatter) - bb_parser.set_defaults() - random_parser = subparsers.add_parser("generate",parents=[common], - description=textwrap.dedent(header + '''\ +""" + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + bb_parser.set_defaults() + random_parser = subparsers.add_parser( + "generate", + parents=[common], + description=textwrap.dedent( + header + + """\ # # # Selected option 2 # # # ############################################################################### -'''),formatter_class=argparse.RawTextHelpFormatter) - random_parser.set_defaults() - bb_sanity = bb_parser.add_argument_group("Sanity checks to avoid faulty entries") - bb_sanity.add_argument( - '--lower-bound', '-l', - type=float, - default=0, - help='ignore entries with duration smaller than this') - bb_sanity.add_argument( - '--upper-bound', '-u', - type=float, - default=float('inf'), - help='ignore entries with duration smaller than this') - bb_log = bb_parser.add_argument_group("Blackboard log information") - bb_log.add_argument( - '--src-uri', - type=str, - help='The MongoDB URI of the blackboard log connection (default: %(default)s)', - default='mongodb://localhost:27017/') - bb_log.add_argument( - '--src-db', - type=str, - help='The name of the blackboard log database (default: %(default)s)', - default='fflog') - bb_log.add_argument( - '--src-col', - type=str, - help='The name of the blackboard log collection (default: %(default)s)', - default='SkillerInterface.Skiller') - bb_log.add_argument( - '--drop-src-col', - type=bool, - help='Delete the skiller blackboard log collection afterwards', - default=False) +""" + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + random_parser.set_defaults() + bb_sanity = bb_parser.add_argument_group("Sanity checks to avoid faulty entries") + bb_sanity.add_argument( + "--lower-bound", + "-l", + type=float, + default=0, + help="ignore entries with duration smaller than this", + ) + bb_sanity.add_argument( + "--upper-bound", + "-u", + type=float, + default=float("inf"), + help="ignore entries with duration smaller than this", + ) + bb_log = bb_parser.add_argument_group("Blackboard log information") + bb_log.add_argument( + "--src-uri", + type=str, + help="The MongoDB URI of the blackboard log connection (default: %(default)s)", + default="mongodb://localhost:27017/", + ) + bb_log.add_argument( + "--src-db", + type=str, + help="The name of the blackboard log database (default: %(default)s)", + default="fflog", + ) + bb_log.add_argument( + "--src-col", + type=str, + help="The name of the blackboard log collection (default: %(default)s)", + default="SkillerInterface.Skiller", + ) + bb_log.add_argument( + "--drop-src-col", + type=bool, + help="Delete the skiller blackboard log collection afterwards", + default=False, + ) - skill = random_parser.add_argument_group("Skill information") - skill.add_argument( - '--quantity', '-n', - type=int, - help='number of entries to generate', - required=True) - skill.add_argument( - '--skill-name', '-s', - type=str, - help='skill name to generate entries for', - required=True) - skill.add_argument( - '--skill-args', '-a', - type=str, - nargs='+', - action='append', - help=textwrap.dedent('''skill arguments. usage -a <arg-name> <val1> <val2> ... + skill = random_parser.add_argument_group("Skill information") + skill.add_argument( + "--quantity", + "-n", + type=int, + help="number of entries to generate", + required=True, + ) + skill.add_argument( + "--skill-name", + "-s", + type=str, + help="skill name to generate entries for", + required=True, + ) + skill.add_argument( + "--skill-args", + "-a", + type=str, + nargs="+", + action="append", + help=textwrap.dedent( + """skill arguments. usage -a <arg-name> <val1> <val2> ... where val<i> are the possible values of the argument that will be chosen from at random * (placeholder value) if no values are given - ''')) + """ + ), + ) - gauss = random_parser.add_argument_group("Mixed gaussian distribution") - gauss.add_argument( - '--gauss-params', '-g', - type=float, - help='mean and standard deviation (in that order) of a gaussian, repeat this option to add more gaussians', - nargs=2, - required=True, - action='append') - gauss.add_argument( - '--dist-weights', '-w', - type=float, - default=[], - help='Weight of each gauss distribution (default 1)', - nargs='+') - gauss.add_argument( - '--lower-bound', '-l', - type=float, - default=0, - help='clip distribution to a lower bound') - gauss.add_argument( - '--upper-bound', '-u', - type=float, - default=float('inf'), - help='clip distribution to an upper bound') + gauss = random_parser.add_argument_group("Mixed gaussian distribution") + gauss.add_argument( + "--gauss-params", + "-g", + type=float, + help="mean and standard deviation (in that order) of a gaussian, repeat this option to add more gaussians", + nargs=2, + required=True, + action="append", + ) + gauss.add_argument( + "--dist-weights", + "-w", + type=float, + default=[], + help="Weight of each gauss distribution (default 1)", + nargs="+", + ) + gauss.add_argument( + "--lower-bound", + "-l", + type=float, + default=0, + help="clip distribution to a lower bound", + ) + gauss.add_argument( + "--upper-bound", + "-u", + type=float, + default=float("inf"), + help="clip distribution to an upper bound", + ) - visual = random_parser.add_argument_group("Visualization options") - visual.add_argument( - '--bin-size', '-b', - type=int, - help='number of bins to display sampled durations (default: %(default)s)', - default=50) - visual.add_argument( - '--non-interactive', '-y', - action='store_true', - help='skip drawing the sample range') - parser.epilog = "--- Arguments common to all sub-parsers ---" \ - + common.format_help().replace(common.format_usage(), '') - random_parser.epilog = ''' + visual = random_parser.add_argument_group("Visualization options") + visual.add_argument( + "--bin-size", + "-b", + type=int, + help="number of bins to display sampled durations (default: %(default)s)", + default=50, + ) + visual.add_argument( + "--non-interactive", + "-y", + action="store_true", + help="skip drawing the sample range", + ) + parser.epilog = ( + "--- Arguments common to all sub-parsers ---" + + common.format_help().replace(common.format_usage(), "") + ) + random_parser.epilog = """ example call: ./mongodb_skillsim_lookup.py generate -d -n \ 1000 -g 10 2 -g 20 3 -w 1 5 -s test -a arg1 value1 value2 -a arg2 - ''' - args = parser.parse_args(args=None if sys.argv[1:] else ['--help']) - # validate inputs - if args==None: - parser.exit(1) + """ + args = parser.parse_args(args=None if sys.argv[1:] else ["--help"]) + # validate inputs + if args == None: + parser.exit(1) - mongoIf = MongoInterface(args.mongodb_uri,args.db,args.collection,args.dry_run) - if(args.drop_collection_first and not args.dry_run): - print("Drop collection before uploading...") - drop_collection(args.mongodb_uri,args.db,args.collection) - if args.subparser == 'bblog': - mongoIf.transform(args.src_uri,args.src_db,args.src_col, - args.lower_bound,args.upper_bound) - if args.drop_src_col: - drop_collection(args.src_mongodb_uri, args.src_db, args.src_col) - elif args.subparser == 'generate': - sampler = GaussSampler(args.quantity, - args.dist_weights, - args.gauss_params, - args.upper_bound, - args.lower_bound); - if(not args.non_interactive): - sampler.display(args.bin_size) - mongoIf.upload(sampler.samples, args.skill_name, args.skill_args) - else: - print("unrecognized mode") + mongoIf = MongoInterface(args.mongodb_uri, args.db, args.collection, args.dry_run) + if args.drop_collection_first and not args.dry_run: + print("Drop collection before uploading...") + drop_collection(args.mongodb_uri, args.db, args.collection) + if args.subparser == "bblog": + mongoIf.transform( + args.src_uri, args.src_db, args.src_col, args.lower_bound, args.upper_bound + ) + if args.drop_src_col: + drop_collection(args.src_mongodb_uri, args.src_db, args.src_col) + elif args.subparser == "generate": + sampler = GaussSampler( + args.quantity, + args.dist_weights, + args.gauss_params, + args.upper_bound, + args.lower_bound, + ) + if not args.non_interactive: + sampler.display(args.bin_size) + mongoIf.upload(sampler.samples, args.skill_name, args.skill_args) + else: + print("unrecognized mode") <<<<<<< HEAD +<<<<<<< HEAD if __name__ == '__main__': main() ======= @@ -398,3 +487,7 @@ if __name__ == "__main__": """ Invoke the main function. """ main() >>>>>>> d55b351fd (fixup! scripts: add docstring to all functions and classes) +======= +if __name__ == "__main__": + main() +>>>>>>> 4db78296e (scripts: reformat lookup script with black) _______________________________________________ fawkes-commits mailing list [email protected] https://lists.kbsg.rwth-aachen.de/listinfo/fawkes-commits
