Skip to content

Commit

Permalink
Adding programmatic tagging for experiment initiation
Browse files Browse the repository at this point in the history
  • Loading branch information
John Hawkins authored and John Hawkins committed Jun 29, 2023
1 parent 1fb75d2 commit 4fea068
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 11 deletions.
8 changes: 6 additions & 2 deletions projit/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,9 +329,13 @@ def task_tag(project, asset, name, values):
Add tags to an asset in the project from the command line
"""
vals = values.split(",")
#print(f"Tagging {asset}:{name} with {vals}" )
tags = {}
for val in vals:
temp = val.split("=")
tags[temp[0]] = temp[1]

if project.validate_asset(asset, name):
project.add_tags(asset, name, vals)
project.add_tags(asset, name, tags)
else:
print(f"ERROR: Invalid request to tag asset {name} of type {asset} - please check available assets")
exit(1)
Expand Down
53 changes: 44 additions & 9 deletions projit/projit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,15 @@ def __init__(self,
self.executions = executions
self.tags = tags


def get_root_path(self):
"""
Get the path to where the project folder is located
"""
return self.path[0:len(self.path) - len(config_folder)]

def start_experiment(self, name, path, params={}):

def start_experiment(self, name, path, params={}, tags={}):
"""
Start an experiment execution.
This function will create a new experiment if this is the first execution
Expand All @@ -132,11 +134,17 @@ def start_experiment(self, name, path, params={}):
:type path: string, required
:param params: Optional dictionary of parameters used in the experiment execution
:type path: Dictionary, option
:type params: Dictionary, optional
:param tags: Optional dictionary of tags to describe the experiment
:type tags: Dictionary, optional
:return: id : The Execution ID
:rtype: String
"""
self.initiate_lock()
self.reload()

if not self.experiment_exists(name):
self.add_experiment(name, path)

Expand All @@ -151,16 +159,19 @@ def start_experiment(self, name, path, params={}):
payload = {'start':startdt, 'end':"", 'githash':ghash, 'params':params}
exper_execs = {}

self.initiate_lock()
self.reload()
if name in self.executions:
exper_execs = self.executions[name]
exper_execs[id] = payload
self.executions[name] = exper_execs
self.save()
self.release_lock()

if len(tags)>0:
self.add_tags("experiment", name, tags)

return id


def end_experiment(self, name, id, hyperparams={}):
"""
End an experiment execution.
Expand Down Expand Up @@ -216,13 +227,15 @@ def get_experiment_execution_stats(self, name):
else:
return 0, 0


def get_mean_execution_time(self, name):
exec_times = self.get_execution_times(name)
if len(exec_times) > 0:
return np.mean(exec_times)
else:
return 0


def get_execution_times(self, name):
if name in self.executions:
exec_times = []
Expand All @@ -236,6 +249,7 @@ def get_execution_times(self, name):
else:
return []


def add_experiment(self, name, path):
"""
Add information of a new experiment to the project.
Expand All @@ -261,7 +275,8 @@ def add_experiment(self, name, path):
self.experiments.append( (name, path) )
self.save()
self.release_lock()



def update_name_description(self, name, descrip):
"""
Update the core values name and description
Expand All @@ -273,6 +288,7 @@ def update_name_description(self, name, descrip):
self.save()
self.release_lock()


def dataset_exists(self, name):
"""
Check if a given dataset is in the data structure
Expand All @@ -288,6 +304,7 @@ def dataset_exists(self, name):
return True
return False


def experiment_exists(self, name):
"""
Check if a given experiment is in the data structure
Expand All @@ -303,6 +320,7 @@ def experiment_exists(self, name):
return True
return False


def validate_asset(self, asset, name):
if asset=="experiment":
return self.experiment_exists(name)
Expand All @@ -311,7 +329,8 @@ def validate_asset(self, asset, name):
else:
return False

def add_tags(self, asset, name, vals):

def add_tags(self, asset, name, tags):
self.initiate_lock()
self.reload()
assets = {}
Expand All @@ -320,14 +339,14 @@ def add_tags(self, asset, name, vals):
assets = self.tags[asset]
if name not in assets:
assets[name] = {}
for val in vals:
split = val.split("=")
assets[name][split[0]] = split[1]
for tag in tags:
assets[name][tag] = tags[tag]

self.tags[asset] = assets
self.save()
self.release_lock()


def get_tags(self, asset, name, tags):
if asset in self.tags:
assets = self.tags[asset]
Expand All @@ -345,6 +364,7 @@ def get_tags(self, asset, name, tags):
else:
return ["" for t in tags]


def clean_experimental_results(self, name):
"""
Remove all results for a given experiment
Expand All @@ -361,6 +381,7 @@ def clean_experimental_results(self, name):
if name in self.dataresults[dataset]:
del self.dataresults[dataset][name]


def add_dataset(self, name, path):
"""
Add a named dataset to the project.
Expand All @@ -380,6 +401,7 @@ def add_dataset(self, name, path):
self.save()
self.release_lock()


def rm_dataset(self, name):
"""
Remove a named dataset to the project.
Expand All @@ -401,6 +423,7 @@ def rm_dataset(self, name):
self.save()
self.release_lock()


def rm_experiment(self, name):
"""
Remove a named experiment from the project.
Expand Down Expand Up @@ -446,6 +469,7 @@ def add_param(self, name, value):
self.save()
self.release_lock()


def add_hyperparam(self, name, value):
"""
Add a set of hyper parameters to the project.
Expand All @@ -468,6 +492,7 @@ def add_hyperparam(self, name, value):
else:
raise Exception("Projit Experiment Exception: No experiment called: '%s' -- Register your experiment first." % name)


def add_result(self, experiment, metric, value, dataset=None):
"""
Add results from an experiment to the project.
Expand Down Expand Up @@ -514,6 +539,7 @@ def add_result(self, experiment, metric, value, dataset=None):
self.save()
self.release_lock()


def get_results(self, dataset=None):
"""
Retrieve the experimental results as a DataFrame.
Expand Down Expand Up @@ -550,6 +576,7 @@ def get_results(self, dataset=None):
cols.extend(rest)
return df.loc[:,cols]


def get_dataset(self, name):
"""
Retrieve the dataset by name.
Expand All @@ -565,25 +592,29 @@ def get_dataset(self, name):
else:
raise Exception("Projit Dataset Exception: Named dataset '%s' not available. Register your dataset" % name)


def get_param(self, name):
if name in self.params:
return self.params[name]
else:
raise Exception("Projit Parameter Exception: Named parameter '%s' is not available:" % name)


def get_hyperparam(self, name):
if name in self.hyperparams:
return self.hyperparams[name]
else:
raise Exception("Projit Parameter Exception: Hyper parameters for experiemnt '%s' are not available:" % name)


def get_path_to_dataset(self, name):
ds = self.get_dataset(name)
if self.is_complete_path(ds):
return ds
else:
return self.create_local_path(ds)


def is_complete_path(self, path):
if path[0:1] == "/":
return True
Expand All @@ -593,6 +624,7 @@ def is_complete_path(self, path):
return True
return False


def create_local_path(self, ds):
return self.get_root_path() + ds

Expand All @@ -614,6 +646,7 @@ def initiate_lock(self):
with open(path_to_lock, 'w') as outfile:
json.dump(lock_content, outfile, indent=0)


def release_lock(self):
"""
Lock files are used during processes that modify the project
Expand Down Expand Up @@ -719,10 +752,12 @@ def load(config_path):
_object.path = config_path
return _object


##########################################################################################
def projit_load():
return load( locate_projit_config() )


##########################################################################################
def init(template, name, desc=""):
"""
Expand Down

0 comments on commit 4fea068

Please sign in to comment.