From 67258b58a48e1211cd9755a0037b6b54f4c9b1c9 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 21 May 2024 21:19:35 -0400 Subject: [PATCH] Adapt for online testing When using tox, pass `-e offline` to exclude online tests. When using pytest, pass `-m "not online"` to do the same. --- pytest.ini | 4 ++ tests/test_manifest.py | 1 + tests/test_probes.py | 129 +++++++++++++++++++++-------------------- tox.ini | 7 +++ 4 files changed, 78 insertions(+), 63 deletions(-) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..e4bb93a38 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = --strict-markers +markers = + online: mark tests are requiring interest access. \ No newline at end of file diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 0d72c6383..177af21ee 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -19,6 +19,7 @@ def test_validate_manifest_against_local_schema(): validate(instance=jsondat, schema=schemadat) +@pytest.mark.online def test_validate_manifest_against_remote_schema(remote_schema): """Ensures that the manifest matches the remote schema, so as to not unexpectedly break clients.""" json_relative: str = '../sherlock/resources/data.json' diff --git a/tests/test_probes.py b/tests/test_probes.py index 4c7527a2b..39e0ef0b5 100644 --- a/tests/test_probes.py +++ b/tests/test_probes.py @@ -19,77 +19,80 @@ def simple_query(sites_info: dict, site: str, username: str) -> QueryStatus: )[site]['status'].status -# Known positives should only use sites trusted to be reliable and unchanging -@pytest.mark.parametrize('site,username',[ - ('GitLab', 'ppfeister'), - ('AllMyLinks', 'blue'), -]) -def test_known_positives_via_message(sites_info, site, username): - assert simple_query(sites_info=sites_info, site=site, username=username) is QueryStatus.CLAIMED +@pytest.mark.online +class TestLiveTargets: + """Actively test probes against live and trusted targets""" + # Known positives should only use sites trusted to be reliable and unchanging + @pytest.mark.parametrize('site,username',[ + ('GitLab', 'ppfeister'), + ('AllMyLinks', 'blue'), + ]) + def test_known_positives_via_message(self, sites_info, site, username): + assert simple_query(sites_info=sites_info, site=site, username=username) is QueryStatus.CLAIMED -# Known positives should only use sites trusted to be reliable and unchanging -@pytest.mark.parametrize('site,username',[ - ('GitHub', 'ppfeister'), - ('GitHub', 'sherlock-project'), - ('Docker Hub', 'ppfeister'), - ('Docker Hub', 'sherlock'), -]) -def test_known_positives_via_status_code(sites_info, site, username): - assert simple_query(sites_info=sites_info, site=site, username=username) is QueryStatus.CLAIMED + # Known positives should only use sites trusted to be reliable and unchanging + @pytest.mark.parametrize('site,username',[ + ('GitHub', 'ppfeister'), + ('GitHub', 'sherlock-project'), + ('Docker Hub', 'ppfeister'), + ('Docker Hub', 'sherlock'), + ]) + def test_known_positives_via_status_code(self, sites_info, site, username): + assert simple_query(sites_info=sites_info, site=site, username=username) is QueryStatus.CLAIMED -# Known positives should only use sites trusted to be reliable and unchanging -@pytest.mark.parametrize('site,username',[ - ('BodyBuilding', 'blue'), - ('labpentestit', 'CSV'), -]) -def test_known_positives_via_response_url(sites_info, site, username): - assert simple_query(sites_info=sites_info, site=site, username=username) is QueryStatus.CLAIMED + # Known positives should only use sites trusted to be reliable and unchanging + @pytest.mark.parametrize('site,username',[ + ('BodyBuilding', 'blue'), + ('labpentestit', 'CSV'), + ]) + def test_known_positives_via_response_url(self, sites_info, site, username): + assert simple_query(sites_info=sites_info, site=site, username=username) is QueryStatus.CLAIMED -# Randomly generate usernames of high length and test for positive availability -# Randomly generated usernames should be simple alnum for simplicity and high -# compatibility. Several attempts may be made ~just in case~ a real username is -# generated. -@pytest.mark.parametrize('site,random_len',[ - ('GitLab', 255), - ('Codecademy', 30) -]) -def test_likely_negatives_via_message(sites_info, site, random_len): - num_attempts: int = 3 - attempted_usernames: list[str] = [] - status: QueryStatus = QueryStatus.CLAIMED - for i in range(num_attempts): - acceptable_types = string.ascii_letters + string.digits - random_handle = ''.join(random.choice(acceptable_types) for _ in range (random_len)) - attempted_usernames.append(random_handle) - status = simple_query(sites_info=sites_info, site=site, username=random_handle) - if status is QueryStatus.AVAILABLE: - break - assert status is QueryStatus.AVAILABLE, f"Could not validate available username after {num_attempts} attempts with randomly generated usernames {attempted_usernames}." + # Randomly generate usernames of high length and test for positive availability + # Randomly generated usernames should be simple alnum for simplicity and high + # compatibility. Several attempts may be made ~just in case~ a real username is + # generated. + @pytest.mark.parametrize('site,random_len',[ + ('GitLab', 255), + ('Codecademy', 30) + ]) + def test_likely_negatives_via_message(self, sites_info, site, random_len): + num_attempts: int = 3 + attempted_usernames: list[str] = [] + status: QueryStatus = QueryStatus.CLAIMED + for i in range(num_attempts): + acceptable_types = string.ascii_letters + string.digits + random_handle = ''.join(random.choice(acceptable_types) for _ in range (random_len)) + attempted_usernames.append(random_handle) + status = simple_query(sites_info=sites_info, site=site, username=random_handle) + if status is QueryStatus.AVAILABLE: + break + assert status is QueryStatus.AVAILABLE, f"Could not validate available username after {num_attempts} attempts with randomly generated usernames {attempted_usernames}." -# Randomly generate usernames of high length and test for positive availability -# Randomly generated usernames should be simple alnum for simplicity and high -# compatibility. Several attempts may be made ~just in case~ a real username is -# generated. -@pytest.mark.parametrize('site,random_len',[ - ('GitHub', 39), - ('Docker Hub', 30) -]) -def test_likely_negatives_via_status_code(sites_info, site, random_len): - num_attempts: int = 3 - attempted_usernames: list[str] = [] - status: QueryStatus = QueryStatus.CLAIMED - for i in range(num_attempts): - acceptable_types = string.ascii_letters + string.digits - random_handle = ''.join(random.choice(acceptable_types) for _ in range (random_len)) - attempted_usernames.append(random_handle) - status = simple_query(sites_info=sites_info, site=site, username=random_handle) - if status is QueryStatus.AVAILABLE: - break - assert status is QueryStatus.AVAILABLE, f"Could not validate available username after {num_attempts} attempts with randomly generated usernames {attempted_usernames}." + # Randomly generate usernames of high length and test for positive availability + # Randomly generated usernames should be simple alnum for simplicity and high + # compatibility. Several attempts may be made ~just in case~ a real username is + # generated. + @pytest.mark.parametrize('site,random_len',[ + ('GitHub', 39), + ('Docker Hub', 30) + ]) + def test_likely_negatives_via_status_code(self, sites_info, site, random_len): + num_attempts: int = 3 + attempted_usernames: list[str] = [] + status: QueryStatus = QueryStatus.CLAIMED + for i in range(num_attempts): + acceptable_types = string.ascii_letters + string.digits + random_handle = ''.join(random.choice(acceptable_types) for _ in range (random_len)) + attempted_usernames.append(random_handle) + status = simple_query(sites_info=sites_info, site=site, username=random_handle) + if status is QueryStatus.AVAILABLE: + break + assert status is QueryStatus.AVAILABLE, f"Could not validate available username after {num_attempts} attempts with randomly generated usernames {attempted_usernames}." def test_username_illegal_regex(sites_info): diff --git a/tox.ini b/tox.ini index 58371324f..0c309410b 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,13 @@ commands = coverage run --source=sherlock --module pytest -v coverage report --show-missing +[testenv:offline] +deps = + jsonschema + pytest +commands = + pytest -v -m "not online" + [testenv:lint] description = Lint with Ruff deps =