Working on tag implementation - modifying CLI to add tags, list asset…

…s with tags

projit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.11"
+__version__ = "0.1.12"
 
 from .utils import locate_projit_config
 from .projit import projit_load

projit/cli.py

@@ -65,6 +65,7 @@ def task_status(project):
     print("  Description: %s" % project.desc)
     print("  Datasets: %i" % len(project.datasets))
     print("  Experiments: %i" % len(project.experiments))
+    print("  Executions: %i" % len(project.executions))
     print("")
 
 ##########################################################################################
@@ -73,7 +74,7 @@ def filler(current, max_len, content=" "):
 
 ##########################################################################################
 def print_header(header):
-    full_header = header + ("_" * (80-len(header)))
+    full_header = header + ("_" * (90-len(header)))
     print(full_header)
 
 ##########################################################################################
@@ -111,38 +112,112 @@ def task_compare(project, datasets, metric, markdown):
         print(results)
 
 
+def extract_max_tags_lengths(project, asset, tags):
+    if asset in project.tags:
+        tagset = project.tags[asset]
+        max_tag_lengths = []
+        for t in tags:
+            temp = []
+            for a in tagset:
+                if t in tagset[a]:
+                    temp.append(len(tagset[a][t]))
+                else:
+                    temp.append(0)
+            max_val = max(temp)
+            if max_val<len(t):
+               max_val = len(t)
+            max_tag_lengths.append(max_val)
+        return max_tag_lengths
+    else:
+        return [0 for x in tags]
+
+
 ##########################################################################################
-def task_list(subcmd, project, dataset, markdown):
+def task_list(subcmd, project, dataset, markdown, tags):
     """
     List content of a project from the command line
     """
+    if len(tags) > 0:
+        tags_max_len = max([len(x) for x in tags])
+    else:
+        tags = []
+        tags_max_len = 0
+
     print()
     if subcmd == "datasets":
         print_header("__Datasets")
         if len(project.datasets.keys()) > 0:
+            tag_header = ""
+            if len(tags)>0:
+               tag_max_lengths = extract_max_tags_lengths(project, "dataset", tags)
+               for tag,tag_len in zip(tags,tag_max_lengths):
+                   tag_header = tag_header + tag + filler(len(tag), tag_len+3, "_")
+
             long_key = max([len(k) for k in project.datasets.keys()])
-            myhead = "__Name" + filler(len("Name"), long_key+3, "_") + "Path_"
+            myhead = "__Name" + filler(len("Name"), long_key+3, "_") + tag_header + "Path_________"
             print_header(myhead)
             for ds in project.datasets:
-                print("  ", ds, filler(len(ds), long_key+3 ), project.datasets[ds], sep="" )
+                tag_output = ""
+                if len(tags)>0:
+                    tag_vals = project.get_tags("dataset", ds, tags)                
+                    for tag,tag_len in zip(tag_vals,tag_max_lengths):
+                        tag_output = tag_output + tag + filler(len(tag), tag_len+3, " ")
+                print("  ", ds, filler(len(ds), long_key+3 ), tag_output, project.datasets[ds], sep="" )
         else:
             print(" NONE")
         print("")
     elif subcmd == "experiments":
         print_header("__Experiments")
         if len(project.experiments) > 0:
+            tag_header = ""
+            if len(tags)>0:
+               tag_max_lengths = extract_max_tags_lengths(project, "experiment", tags)
+               for tag,tag_len in zip(tags,tag_max_lengths):
+                   tag_header = tag_header + tag + filler(len(tag), tag_len+3, "_")
+
             long_key = max([len(k[0]) for k in project.experiments])
-            myhead = "__Name__" + filler(len("Name__"), long_key+3, "_") + "Runs__" + "MeanTime____" + "Path"
+            myhead = "__Name__" + filler(len("Name__"), long_key+3, "_") + tag_header + "Runs__" + "MeanRunTime___" + "Path______"
             print_header(myhead)
             for exp in project.experiments:
+                tag_output = ""
+                if len(tags)>0:
+                    tag_vals = project.get_tags("experiment", exp[0], tags)
+                    for tag,tag_len in zip(tag_vals,tag_max_lengths):
+                        tag_output = tag_output + tag + filler(len(tag), tag_len+3, " ")
                 execs, mean_time = project.get_experiment_execution_stats(exp[0])
                 mins, secs = divmod(mean_time, 60)
-                if mins>0:
-                    mytime = f"{int(mins)}M {int(secs)}s"
+                if mins>60:
+                    hours, mins = divmod(mins, 60)
                 else:
-                    mytime = f"{int(secs)}s"
-                print("  ", exp[0], filler(len(exp[0]), long_key+3), 
-                           execs, filler(len(str(execs)), 6), 
+                    hours = 0
+                hours = int(hours)
+                mins = int(mins)
+                secs = int(secs)
+                if hours>9:
+                    h_str = f"{hours}h"
+                elif hours==0:
+                    h_str = f"   "
+                else:
+                    h_str = f" {hours}h"
+
+                if mins>9:
+                    m_str = f"{mins}m"
+                elif mins==0:
+                    m_str = f"   "
+                else:
+                    m_str = f" {mins}m"
+
+                if secs>9:
+                    s_str = f"{secs}s"
+                elif mins==0:
+                    s_str = f"   "
+                else:
+                    s_str = f" {secs}s"
+
+                mytime = f" {h_str} {m_str} {s_str}  "
+
+                print("  ", exp[0], filler(len(exp[0]), long_key+3), tag_output, 
+                           filler(len(str(execs)), 4), execs, "  ", 
                            mytime, filler(len(str(mytime)), 12), 
                            exp[1], sep=""  
                 )
@@ -248,6 +323,20 @@ def task_add(project, asset, name, path):
         print("ERROR: Request to add unrecognised asset type: %s" % asset)
         exit(1)
 
+##########################################################################################
+def task_tag(project, asset, name, values):
+    """
+    Add tags to an asset in the project from the command line
+    """
+    vals = values.split(",")
+    #print(f"Tagging {asset}:{name} with {vals}" )
+    if project.validate_asset(asset, name):
+        project.add_tags(asset, name, vals)
+    else: 
+        print(f"ERROR: Invalid request to tag asset {name} of type {asset} - please check available assets")
+        exit(1)
+
+
 ##########################################################################################
 def task_rm(project, asset, name):
     """
@@ -363,9 +452,15 @@ def cli_main():
    add_parser.add_argument('name')
    add_parser.add_argument('path')
 
+   add_parser = subparsers.add_parser('tag')
+   add_parser.add_argument('asset')
+   add_parser.add_argument('name')
+   add_parser.add_argument('values')
+
    list_parser = subparsers.add_parser('list')
    list_parser.add_argument('subcmd')
    list_parser.add_argument('dataset', nargs='?', default="")
+   list_parser.add_argument('--tags', nargs='+', default="")
 
    plot_parser = subparsers.add_parser('plot')
    plot_parser.add_argument('experiment')
@@ -412,7 +507,7 @@ def cli_main():
    project = projit_load(config_path)
 
    if args.cmd == 'list':
-      task_list(args.subcmd, project, args.dataset, args.markdown)
+      task_list(args.subcmd, project, args.dataset, args.markdown, args.tags)
 
    if args.cmd == 'compare':
       datasets = args.datasets.split(",")
@@ -421,6 +516,9 @@ def cli_main():
    if args.cmd == 'add':
       task_add(project, args.asset, args.name, args.path)
 
+   if args.cmd == 'tag':
+      task_tag(project, args.asset, args.name, args.values)
+
    if args.cmd == 'rm':
       task_rm(project, args.asset, args.name)
 

projit/config.py

@@ -5,4 +5,5 @@
 data_file = "data.json"
 experiments_file = "experiments.json"
 execution_file = "executions.json"
+tag_file = "tags.json"
 lock_file = "LOCK"

projit/projit.py

@@ -11,6 +11,7 @@
 from .config import lock_file
 from .config import config_file
 from .config import execution_file
+from .config import tag_file
 from .config import config_folder
 from .template import load_template
 from .utils import locate_projit_config
@@ -37,7 +38,9 @@ def __init__(self,
                  params={},
                  hyperparams={},
                  dataresults={},
-                 executions={}):
+                 executions={},
+                 tags={}
+    ):
         """
         Initialise a projit project object.
         This class will be used for storing and retrieving all data about the project,
@@ -90,6 +93,9 @@ def __init__(self,
                                            }
                                        }
         :type executions: Dictionary of Dictionary of Dictionary, optional
+ 
+        :param tags: The dictionary of tags for project assets.
+        :type tags: Dictionary of Dictionary of Dictionary, optional
 
         :return: None 
         :rtype: None 
@@ -104,7 +110,7 @@ def __init__(self,
         self.hyperparams = hyperparams
         self.dataresults = dataresults
         self.executions = executions
-
+        self.tags = tags
 
     def get_root_path(self):
         """
@@ -267,6 +273,21 @@ def update_name_description(self, name, descrip):
         self.save()
         self.release_lock()
 
+    def dataset_exists(self, name):
+        """
+        Check if a given dataset is in the data structure
+
+        :param name: The dataset name
+        :type name: string, required
+
+        :return: exists
+        :rtype: Boolean
+        """
+        for elem in self.datasets:
+            if elem == name:
+                return True
+        return False
+
     def experiment_exists(self, name):
         """
         Check if a given experiment is in the data structure
@@ -281,6 +302,48 @@ def experiment_exists(self, name):
             if elem[0] == name:
                 return True
         return False
+
+    def validate_asset(self, asset, name):
+        if asset=="experiment":
+            return self.experiment_exists(name)
+        elif asset=="dataset":
+            return self.dataset_exists(name)
+        else:
+            return False
+
+    def add_tags(self, asset, name, vals):
+        self.initiate_lock()
+        self.reload()
+        assets = {}
+        assets[name] = {}
+        if asset in self.tags:
+            assets = self.tags[asset]
+            if name not in assets:
+                assets[name] = {}
+        for val in vals:
+            split = val.split("=")
+            assets[name][split[0]] = split[1]
+
+        self.tags[asset] = assets
+        self.save()
+        self.release_lock()
+
+    def get_tags(self, asset, name, tags):
+        if asset in self.tags:
+            assets = self.tags[asset]
+            if name not in assets:
+                return ["" for t in tags]
+            else:
+                my_asset = assets[name]
+                tag_set = []
+                for t in tags:
+                    if t in my_asset:
+                        tag_set.append(my_asset[t])
+                    else:
+                        tag_set.append("")
+                return tag_set
+        else:
+            return ["" for t in tags]
 
     def clean_experimental_results(self, name):
         """
@@ -568,6 +631,7 @@ def save(self):
         """
         core_props = self.__dict__.copy()
         del core_props['executions']
+        del core_props['tags']
         path_to_json = self.path + "/" + config_file
         with open(path_to_json, 'w') as outfile:
             json.dump(core_props, outfile, indent=0)
@@ -576,13 +640,19 @@ def save(self):
         with open(path_to_json, 'w') as outfile:
             json.dump(self.executions, outfile, indent=0)
 
+        path_to_json = self.path + "/" + tag_file
+        with open(path_to_json, 'w') as outfile:
+            json.dump(self.tags, outfile, indent=0)
+
+
     def reload(self):
         """
         Sometimes we reload the project from disk. Necessary when multiple processes are running
         experiments in the same project.
         """
         path_to_config = self.path + "/" + config_file
         path_to_execs = self.path + "/" + execution_file
+        path_to_tags = self.path + "/" + tag_file
         _dict = {}
         if os.path.exists(path_to_config):
             with open(path_to_config) as f:
@@ -596,6 +666,13 @@ def reload(self):
                 _execs = json.load(f)
                 setattr(self, "executions", _execs)
 
+        _tags = {}
+        if os.path.exists(path_to_tags):
+            with open(path_to_tags) as f:
+                _tags = json.load(f)
+                setattr(self, "tags", _tags)
+
+
     def render(self, path):
         results = self.get_results()
         pdf = PDF()
@@ -632,7 +709,13 @@ def load(config_path):
         with open(path_to_execs) as f:
             _execs['executions'] = json.load(f)
 
-    _object = Projit(**_dict, **_execs )
+    _tags = {}
+    path_to_tags = config_path + "/" + tag_file
+    if os.path.exists(path_to_tags):
+        with open(path_to_tags) as f:
+            _tags["tags"] = json.load(f)
+
+    _object = Projit(**_dict, **_execs, **_tags )
     _object.path = config_path
     return _object