diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 51fc0a3..fe1680c 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -230,7 +230,23 @@ You can then list the results just for that specific dataset: +Compare Results +^^^^^^^^^^^^^^^^^^^^^ + +You can compare results across dataset using a simple CLI option. +The syntax requires that you provide a comma separates list of datasets as well as the metric you want to +use for the comparison. If you want to compare multiple metrics you will need to create multiple tables. + +For example + +.. code-block:: bash + + >projit compare dataset1,dataset2,dataset3 RMSE + +Will produce a table where each row corresponds to a specific experiment, and each column will correspond +to one of the three specfied datasets. Within the table each cell will contain the RMSE of the experiment +on that dataset. diff --git a/projit/__init__.py b/projit/__init__.py index 4a7d855..4e8b1aa 100644 --- a/projit/__init__.py +++ b/projit/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.10" +__version__ = "0.1.11" from .utils import locate_projit_config from .projit import projit_load diff --git a/projit/cli.py b/projit/cli.py index 0175197..424e699 100644 --- a/projit/cli.py +++ b/projit/cli.py @@ -1,5 +1,6 @@ import argparse import pandas as pd +import numpy as np import sys import os @@ -81,18 +82,24 @@ def task_compare(project, datasets, metric, markdown): for the specified metric to compile the comparison dataset to display. """ title = "Compare Results" + warning = "" results = None for dataset in datasets: rez = project.get_results(dataset) if metric not in rez.columns: rez[metric] = np.nan + warning += f"Metric '{metric}' not present for dataset '{dataset}'\n" rez = rez.loc[:,['experiment',metric]] rez.columns = ['experiment', dataset] if results is None: results = rez else: - results = pd.merge(result,rez,on="experiment") + results = pd.merge(results,rez,on="experiment") + if len(warning) > 0: + print("*** WARNINGS ***") + print(warning) + if markdown: print_results_markdown(title, results) else: @@ -323,9 +330,16 @@ def print_usage(prog): print("") - ########################################################################################## def main(): + try: + cli_main() + except Exception as e: + print("*** Projit CLI Error ***") + print(e) + +########################################################################################## +def cli_main(): parser = argparse.ArgumentParser() parser.add_argument('-v', '--version', help='Print Version', action='store_true') parser.add_argument('-m', '--markdown', help='Use markdown for output', action='store_true') @@ -394,7 +408,7 @@ def main(): if args.cmd == 'list': task_list(args.subcmd, project, args.dataset, args.markdown) - if args.cmd == 'list': + if args.cmd == 'compare': datasets = args.datasets.split(",") task_compare(project, datasets, args.metric, args.markdown) diff --git a/projit/projit.py b/projit/projit.py index a40ba80..bc11875 100644 --- a/projit/projit.py +++ b/projit/projit.py @@ -169,17 +169,17 @@ def end_experiment(self, name, id, hyperparams={}): """ if not self.experiment_exists(name): - raise Exception(f"ERROR: Cannot end experiment: '{name}' -- Experiment not registered") + raise Exception(f"Projit Experiment Exception: Cannot end experiment: '{name}' -- Experiment not registered") if name in self.executions: exper_execs = self.executions[name] else: - raise Exception(f"ERROR: Cannot end experiment: '{name}' -- Executions not started") + raise Exception(f"Projit Experiment Exception: Cannot end experiment: '{name}' -- Executions not started") if id in exper_execs: payload = exper_execs[id] else: - raise Exception(f"ERROR: Cannot end experiment: '{name}' -- Executions not started") + raise Exception(f"Projit Experiment Exception: Cannot end experiment: '{name}' -- Executions not started") payload['end'] = str(datetime.now()) payload['hyperparams'] = hyperparams @@ -367,7 +367,7 @@ def add_hyperparam(self, name, value): self.hyperparams[name] = value self.save() else: - raise Exception("ERROR: No experiment called: '%s' -- Register your experiment first." % name) + raise Exception("Projit Experiment Exception: No experiment called: '%s' -- Register your experiment first." % name) def add_result(self, experiment, metric, value, dataset=None): """ @@ -432,7 +432,7 @@ def get_results(self, dataset=None): if dataset in self.dataresults: myresults = self.dataresults[dataset] else: - raise Exception("ERROR: No results for dataset: %s " % dataset) + raise Exception("Projit Dataset Exception: No results for dataset: %s " % dataset) for exp in self.experiments: key = exp[0] if key in myresults: @@ -461,19 +461,19 @@ def get_dataset(self, name): if name in self.datasets: return self.datasets[name] else: - raise Exception("ERROR: Named dataset '%s' not available:" % name) + raise Exception("Projit Dataset Exception: Named dataset '%s' not available. Register your dataset" % name) def get_param(self, name): if name in self.params: return self.params[name] else: - raise Exception("ERROR: Named parameter '%s' is not available:" % name) + raise Exception("Projit Parameter Exception: Named parameter '%s' is not available:" % name) def get_hyperparam(self, name): if name in self.hyperparams: return self.hyperparams[name] else: - raise Exception("ERROR: Hyper parameters for experiemnt '%s' are not available:" % name) + raise Exception("Projit Parameter Exception: Hyper parameters for experiemnt '%s' are not available:" % name) def get_path_to_dataset(self, name): ds = self.get_dataset(name)