Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add date as an argument in the replay.user mode in the hlt client #426

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions tools/hlt_client/hlt_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ def _parse_arguments():
help='Number of replays to fetch')
replay_user_parser.add_argument('-d', '--destination', dest='destination', action='store', type=str, required=True,
help="In which folder to store all resulting replay files.")
replay_user_parser.add_argument('-t', '--date', action='store', type=str, dest='date', required=True,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should not be a required argument.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, I will change it to 'default=None' and remove 'required=True'.

help="Fetch replay files matching the specified date. To fetch a day's files user"
"the YYYYMMDD format.")
# .Modes.Replay.Modes.Date
replay_regex_parser = replay_subparser.add_parser(REPLAY_MODE_DATE, help='Retrieve replays based on regex')
replay_regex_parser.add_argument('-t', '--date', action='store', type=str, dest='date', required=True,
Expand Down
27 changes: 20 additions & 7 deletions tools/hlt_client/hlt_client/download_game.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,30 +128,41 @@ class UserGameDownloader(GameDownloader):
_FETCH_THRESHOLD = 250
_BUCKETS = []

def __init__(self, destination, user_id, limit):
def __init__(self, destination, user_id, limit, date):
"""
Download games for a user
:param destination: Where to download
:param user_id: Which user's replays to fetch
:param limit: How many replays to fetch (max)
:param date: Which date to download
"""
self.destination = destination
self.objects = self._parse_user_metadata(self._fetch_metadata(user_id, limit))
self.objects = self._parse_user_metadata(self._fetch_metadata(user_id, limit, date))

def _fetch_metadata(self, user_id, limit):
def _fetch_metadata(self, user_id, limit, date):
"""
Retrieves paginated game metadata from the halite servers for a specified user up to limit items
:param user_id: The id of the user to fetch
:param limit: The maximum number of items to fetch
:param date: Which date to download
:return: The full metadata of items
"""
print('Fetching Metadata')
current = 0
result_set = []
while current <= limit:
current_limit = self._FETCH_THRESHOLD if ((limit - current) >= self._FETCH_THRESHOLD) else (limit - current)
result_set += requests.get(self._USER_BOT_URI.format(user_id, current_limit, current)).json()
current += self._FETCH_THRESHOLD
if date is None:
current_limit = self._FETCH_THRESHOLD if ((limit - current) >= self._FETCH_THRESHOLD) else (limit - current)
result_set += requests.get(self._USER_BOT_URI.format(user_id, current_limit, current)).json()
current += self._FETCH_THRESHOLD
else:
if requests.get(self._USER_BOT_URI.format(user_id, 1, current)).json()[0]["replay"].split("-")[1] != date:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Magic number

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was trying to check the date of the retrieved replay one at a time. Do you have any suggestion on how to do this faster? I can also retrieve current_limit replays at a time and filter out replays that belongs to the wrong date and append the filtered list to the list result_set.

current += 1
limit += 1
continue
else:
result_set += requests.get(self._USER_BOT_URI.format(user_id, 1, current)).json()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Magic number

current += 1
print('Finished metadata fetch. Found {} game files.'.format(len(result_set)))
return result_set

Expand Down Expand Up @@ -197,5 +208,7 @@ def download(mode, destination, date, all_bots, default_user_id, user_id, limit)
elif mode == client.REPLAY_MODE_USER:
if not (default_user_id or user_id):
raise ValueError("Cannot run default mode without authenticating .Please run `client.py --auth` first.")
UserGameDownloader(destination, default_user_id if not user_id else user_id, limit).get_objects()
if date != None and not _valid_date(date):
raise ValueError("Date must match format YYYYMMDD")
UserGameDownloader(destination, default_user_id if not user_id else user_id, limit, date).get_objects()
print('Finished writing files to desired location')