diff --git a/.dockerignore b/.dockerignore index 9978ae96e..21bb87de4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,4 @@ tests/ *.txt !/requirements.txt venv/ - +devel/ \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..76001ae9a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,11 @@ +### REPOSITORY +/.github/CODEOWNERS @sdushantha +/LICENSE @sdushantha + +### PACKAGING +# Changes made to these items without code owner approval may negatively +# impact packaging pipelines. Code owners may need time to verify or adapt. +/pyproject.toml @ppfeister @sdushantha +/setup.cfg @ppfeister @sdushantha +/setup.py @ppfeister +/*.spec @ppfeister diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index f7f9b8227..000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Bug report -about: Report a bug in Sherlock's functionality -title: '' -labels: bug -assignees: '' - ---- - - - - -## Checklist - - -- [ ] I'm reporting a bug in Sherlock's functionality -- [ ] The bug I'm reporting is not a false positive or a false negative -- [ ] I've verified that I'm running the latest version of Sherlock -- [ ] I've checked for similar bug reports including closed ones -- [ ] I've checked for pull requests that attempt to fix this bug - -## Description - - -WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 000000000..9dd716553 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,57 @@ +name: Bug Report +description: File a bug report +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: description + attributes: + label: Description + description: Describe the bug you are experiencing + placeholder: Something happend when I did something + validations: + required: true + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to reproduce + description: Write a step by step list that will allow us to reproduce this bug + placeholder: | + 1. Do something + 2. Then do something else + validations: + required: true + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + description: Explain how you expect Sherlock to behave + placeholder: I expect Sherlock to behave like this when that is done + validations: + required: true + - type: textarea + id: actual-behavior + attributes: + label: Actual behavior + description: Explain how Sherlock is acutally behaving + placeholder: Sherlock should be behaving like this when that is done + validations: + required: true + - type: textarea + id: additional-info + attributes: + label: Additional information + description: If you have some additional information, please write it here + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/sherlock-project/sherlock/blob/master/docs/CODE_OF_CONDUCT.md). + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3ba13e0ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/false-negative.yml b/.github/ISSUE_TEMPLATE/false-negative.yml new file mode 100644 index 000000000..03d212293 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/false-negative.yml @@ -0,0 +1,20 @@ +name: False Negative +description: Report a site that is returning false negative results +labels: ["false negative"] +body: + - type: input + id: site-name + attributes: + label: Site name + description: What site is returning false negatives? + placeholder: Reddit + validations: + required: True + - type: textarea + id: additional-info + attributes: + label: Additional info + description: If you know why the site is returning false negatives, please explain + placeholder: Reddit is returning false negatives because... + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/false-positive.yml b/.github/ISSUE_TEMPLATE/false-positive.yml new file mode 100644 index 000000000..e8bc4587c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/false-positive.yml @@ -0,0 +1,20 @@ +name: Flase Positive +description: Report a site that is returning false positive results +labels: ["false positive"] +body: + - type: input + id: site-name + attributes: + label: Site name + description: What site is returning false positives? + placeholder: Reddit + validations: + required: True + - type: textarea + id: additional-info + attributes: + label: Additional info + description: If you know why the site is returning false positives, please explain + placeholder: Reddit is returning false positives because... + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md deleted file mode 100644 index 67ec7ecf0..000000000 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Feature request -about: Request a new functionality for Sherlock -title: '' -labels: enhancement -assignees: '' - ---- - - - -## Checklist - -- [ ] I'm reporting a feature request -- [ ] I've checked for similar feature requests including closed ones - -## Description - - -WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 000000000..654ac7dc1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,12 @@ +name: Feature Request +description: Request a feature +labels: ["enhancement"] +body: + - type: textarea + id: description + attributes: + label: Description + description: Describe the feature you are requesting + placeholder: I'd like Sherlock to be able to do xyz + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md deleted file mode 100644 index 07cc2eec5..000000000 --- a/.github/ISSUE_TEMPLATE/question.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Question -about: Ask us a question -title: '' -labels: question -assignees: '' - ---- - - - -## Checklist - -- [ ] I'm asking a question regarding Sherlock -- [ ] My question is not a tech support question. - -**We are not your tech support**. -If you have questions related to `pip`, `git`, or something that is not related to Sherlock, please ask them on [Stack Overflow](https://stackoverflow.com/) or [r/learnpython](https://www.reddit.com/r/learnpython/) - - -## Question - -ASK YOUR QUESTION HERE diff --git a/.github/ISSUE_TEMPLATE/reporting-false-negative.md b/.github/ISSUE_TEMPLATE/reporting-false-negative.md deleted file mode 100644 index d511c5f86..000000000 --- a/.github/ISSUE_TEMPLATE/reporting-false-negative.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Reporting false negative -about: Reporting a site that is returning false positives -title: '' -labels: false negative -assignees: '' - ---- - - - -## Checklist - -- [ ] I'm reporting a website that is returning **false negative** results -- [ ] I've checked for similar site support requests including closed ones -- [ ] I've checked for pull requests attempting to fix this false negative -- [ ] I'm only reporting **one** site (create a separate issue for each site) - -## Description - - -WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/reporting-false-positive.md b/.github/ISSUE_TEMPLATE/reporting-false-positive.md deleted file mode 100644 index 7f968b5e6..000000000 --- a/.github/ISSUE_TEMPLATE/reporting-false-positive.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Reporting false positive -about: Reporting a site that is returning false positives -title: '' -labels: false positive -assignees: '' - ---- - - - -## Checklist - -- [ ] I'm reporting a website that is returning **false positive** results -- [ ] I've checked for similar site support requests including closed ones -- [ ] I've checked for pull requests attempting to fix this false positive -- [ ] I'm only reporting **one** site (create a separate issue for each site) - -## Description - - -WRITE DESCRIPTION HERE diff --git a/.github/ISSUE_TEMPLATE/site-request.yml b/.github/ISSUE_TEMPLATE/site-request.yml new file mode 100644 index 000000000..86d7b169b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/site-request.yml @@ -0,0 +1,20 @@ +name: Site Support +description: Request a site Sherlock should support +labels: ["site support request"] +body: + - type: input + id: site-name + attributes: + label: Site name + description: What site should Sherlock support? + placeholder: Reddit + validations: + required: True + - type: textarea + id: additional-info + attributes: + label: Additional info + description: If you have suggestions on how Sherlock should detect for usernames, please explain below + placeholder: Sherlock can detect if a username exists on Reddit by checking for... + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/site-support-request.md b/.github/ISSUE_TEMPLATE/site-support-request.md deleted file mode 100644 index 3810c55d6..000000000 --- a/.github/ISSUE_TEMPLATE/site-support-request.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: Site support request -about: Request support for a new site -title: '' -labels: site support request -assignees: '' - ---- - - - -## Checklist - - -- [ ] I'm requesting support for a new site -- [ ] I've checked for similar site support requests including closed ones -- [ ] I've checked that the site I am requesting has not been removed in the past and is not documented in [removed_sites.md](https://github.com/sherlock-project/sherlock/blob/master/removed_sites.md) -- [ ] The site I am requesting support for is not a pornographic website -- [ ] I'm only requesting support of **one** website (create a separate issue for each site) - -## Description - - -URL: diff --git a/.github/workflows/update-site-list.yml b/.github/workflows/update-site-list.yml index 1966beab2..5a0679332 100644 --- a/.github/workflows/update-site-list.yml +++ b/.github/workflows/update-site-list.yml @@ -29,8 +29,8 @@ jobs: python-version: '3.x' # Execute the site_list.py Python script - - name: Execute site_list.py - run: python site_list.py + - name: Execute site-list.py + run: python devel/site-list.py # Commit any changes made by the script - name: Commit files diff --git a/.gitignore b/.gitignore index 0552d4171..4d47421ae 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ __pycache__/ # Pip src/ +# Pip / PyProject Devel & Installation +*.egg-info/ + # Jupyter Notebook .ipynb_checkpoints *.ipynb diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 7a2fa3fe9..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,31 +0,0 @@ -# How To Contribute To Sherlock -First off, thank you for the help! - -There are many ways to contribute. Here is some high level grouping. - -## Adding New Sites - -Please look at the Wiki entry on -[adding new sites](https://github.com/sherlock-project/sherlock/wiki/Adding-Sites-To-Sherlock) -to understand the issues. - -Any new sites that are added need to have a username that has been claimed, and one -that is unclaimed documented in the site data. This allows the regression tests -to ensure that everything is working. - -It is required that a contributor test any new sites by either running the full tests, or running -a site-specific query against the claimed and unclaimed usernames. - -It is not required that a contributor run the -[site_list.py](https://github.com/sherlock-project/sherlock/blob/master/site_list.py) -script. - -If there are performance problems with a site (e.g. slow to respond, unreliable uptime, ...), then -the site may be removed from the list. The -[removed_sites.md](https://github.com/sherlock-project/sherlock/blob/master/removed_sites.md) -file contains sites that were included at one time in Sherlock, but had to be removed for -one reason or another. - -## Adding New Functionality - -Please ensure that the content on your branch passes all tests before submitting a pull request. diff --git a/Dockerfile b/Dockerfile index 4bbdd871c..ca2db2ec7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,9 @@ LABEL org.label-schema.vcs-ref=$VCS_REF \ COPY --from=build /wheels /wheels COPY . /opt/sherlock/ -RUN pip3 install --no-cache-dir -r requirements.txt -f /wheels \ +RUN pip3 install --no-cache-dir . -f /wheels \ && rm -rf /wheels WORKDIR /opt/sherlock/sherlock -ENTRYPOINT ["python", "sherlock.py"] +ENTRYPOINT ["sherlock"] diff --git a/site_list.py b/devel/site-list.py similarity index 85% rename from site_list.py rename to devel/site-list.py index 1f0f05fd4..1b4a03cfb 100644 --- a/site_list.py +++ b/devel/site-list.py @@ -4,7 +4,7 @@ import json # Read the data.json file -with open("sherlock/resources/data.json", "r", encoding="utf-8") as data_file: +with open("../sherlock/resources/data.json", "r", encoding="utf-8") as data_file: data: dict = json.load(data_file) # Removes schema-specific keywords for proper processing @@ -15,7 +15,7 @@ social_networks: list = sorted(social_networks.items()) # Write the list of supported sites to sites.md -with open("sites.md", "w") as site_file: +with open("../sites.md", "w") as site_file: site_file.write(f"## List Of Supported Sites ({len(social_networks)} Sites In Total!)\n") for social_network, info in social_networks: url_main = info["urlMain"] @@ -23,7 +23,7 @@ site_file.write(f"1. ![](https://www.google.com/s2/favicons?domain={url_main}) [{social_network}]({url_main}) {is_nsfw}\n") # Overwrite the data.json file with sorted data -with open("sherlock/resources/data.json", "w") as data_file: +with open("../sherlock/resources/data.json", "w") as data_file: sorted_data = json.dumps(data, indent=2, sort_keys=True) data_file.write(sorted_data) data_file.write("\n") diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 3182120e8..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: '2' - -services: - sherlock: - build: . - volumes: - - "./results:/opt/sherlock/results" diff --git a/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to docs/CODE_OF_CONDUCT.md diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 000000000..996efe8a3 --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,88 @@ +

+
+ +
+

+ +

+ Home +    |    + Installation +    |    + Usage +    |    + Docker +    |    + Contributing +

+ +# How to contribute to Sherlock + +We would love to have you help us with the development of Sherlock. Each and every contribution is greatly valued! + +Here are some things we would appreciate your help on: +1. [Adding targets](#adding-targets) +1. [Cleaning up existing targets](#removing-targets) +1. [Restoring previously removed targets](#restoring-targets) + +## Adding targets + +Please look at the Wiki entry on [adding new sites][wiki_new_sites] to understand the issues. + +All new sites that are added to Sherlock need to have an existing (already claimed) username included in their definition. The linked Wiki page describes this in more detail. This inclusion allows us to run unit tests and prevent regression. + +Contributors are *required* to test any new sites for both false positives and false negatives. Contributors are *encouraged* to run unit tests as well. + +Contributors do not have to run the [site_list.py](/site_list.py) script, as it's ran automagically on master after each manifest change. + +## Removing targets + +If there are performance problems with a site (e.g. slow to respond, unreliable uptime, ...), then +the site may be removed from the list. The [removed_sites.md][file_removed_md] file contains sites that were included at one time in Sherlock, but had to be removed for one reason or another. + +If a site has *occasional* performance problems, but is otherwise accurate, it may be preferable to add a test to weed out false positives rather than removing it. + +## Restoring targets + +Likely our biggest backlog. If you can propose a functional query that complies with [#Adding targets](#adding-targets) that would shrink our [removed sites list][file_removed_md], that would be greatly appreciated. + +## Adding New Functionality + +Contributors that would like to add a feature to Sherlock should open an new [issue][issues_new], proposing their idea. Indicate that you would like to make a Pull Request for said feature. + +Creating an Issue prior to opening a PR helps with tracking, discussions, and avoids hurt feelings if for whatever reason we don't feel that a feature is compatible with the project. + +Please ensure that the content on your branch passes all tests before submitting a pull request. + +# Coverage and Unit Tests + +Thank you for contributing to Sherlock! + +Before creating a pull request with new development, please run the tests +to ensure that everything is working great. It would also be a good idea to run the tests +before starting development to distinguish problems between your +environment and the Sherlock software. + +The following is an example of the command line to run all the tests for +Sherlock. This invocation hides the progress text that Sherlock normally +outputs, and instead shows the verbose output of the tests. + +```console +$ cd sherlock/sherlock +$ python3 -m unittest tests.all --verbose +``` + +Unfortunately, some of the sites that Sherlock checks are not always reliable, so it is common +to get response problems. Any problems in connection will show up as warnings in the tests instead of true errors. + +If some sites are failing due to connection problems (site is down, in maintenance, etc) you can exclude them from tests by creating a `tests/.excluded_sites` file with a list of sites to ignore (one site name per line). + +## Coverage for new features + +Contributors that add new features are *encouraged* make an attempt at creating unit tests for them, as well. Not all contributions are suitable for unit tests, but when it's doable, it helps prevent regression. + + + +[wiki_new_sites]: https://github.com/sherlock-project/sherlock/wiki/Adding-Sites-To-Sherlock +[file_removed_md]: /removed_sites.md +[issues_new]: https://github.com/sherlock-project/sherlock/issues/new/choose \ No newline at end of file diff --git a/docs/INSTALL.md b/docs/INSTALL.md new file mode 100644 index 000000000..9813a09c0 --- /dev/null +++ b/docs/INSTALL.md @@ -0,0 +1,95 @@ +

+
+ +
+

+ +

+ Home +    |    + Installation +    |    + Usage +    |    + Docker +    |    + Contributing +

+ + + +# Alternative install methods + +1. __[Python package](#python)__ + 1. [Build from source](#build-and-install-package-from-source) +1. __[Docker Container](#docker)__ + 1. [via Docker Hub (one off)](#docker) + 1. [via Docker Hub (compose)](#using-compose) + 1. [Local image with Dockerfile](#build-image-from-source-alternative-to-docker-hub) + +

+Python +PyPI - Version +

+ +```bash +# pipx is recommended, but pip may suffice if pipx is unavailable +pipx install sherlock-project +``` + +### Build python package from source (useful for contributors) + +```bash +# pipx is recommended, but pip may suffice if pipx is unavailable +git clone https://github.com/sherlock-project/sherlock.git +cd sherlock +pipx install . +``` + +

+Docker +Docker Image Version +

+ +> [!NOTE] +> Sherlock doesn't yet have context detection. It's recommended that Docker containers be ran with option `-o /opt/sherlock/results/{user123}.txt` (replace {user123}) when an output file is desired at the mounted volume (as seen in the compose). +> +> This has no effect on stdout, which functions as expected out of the box. + +```bash +# One-off searches +docker run --rm -t sherlock/sherlock user123 + +# If you need to save the output file... (modify as needed) +# Output file will land in ${pwd}/results +docker run --rm -t -v "$PWD/results:/opt/sherlock/results" sherlock/sherlock -o /opt/sherlock/results/text.txt user123 +``` + +```bash +# At any time, you may update the image via this command +docker pull sherlock/sherlock +``` + +### Using compose + +```yml +version: "3" +services: + sherlock: + container_name: sherlock + image: sherlock/sherlock + volumes: + - ./sherlock/:/opt/sherlock/results/ +``` + +```bash +docker compose run sherlock user123 +``` + +### Build image from source (useful for contributors) + +```bash +# Assumes ${pwd} is repository root +docker build -t sherlock . +docker run --rm -t sherlock user123 +``` diff --git a/README.md b/docs/README.md similarity index 57% rename from README.md rename to docs/README.md index f8b1c0290..9df30261d 100644 --- a/README.md +++ b/docs/README.md @@ -2,18 +2,18 @@

- Hunt down social media accounts by username across social networks + Hunt down social media accounts by username across 400+ social networks

- Installation + Installation    |    - Usage + Usage    |    - Docker Notes + Docker    |    - Contributing + Contributing

@@ -24,21 +24,27 @@ ## Installation -```console -# clone the repo -$ git clone https://github.com/sherlock-project/sherlock.git +[![PyPI - Version](https://img.shields.io/pypi/v/sherlock-project?logo=PyPi&label=PyPI&color=darkgreen)][ext_pypi] [![Docker Image Version](https://img.shields.io/docker/v/sherlock/sherlock?sort=semver&logo=docker&label=Docker&color=darkgreen)][docs_docker] [![homebrew version](https://img.shields.io/homebrew/v/sherlock?logo=Homebrew&color=darkgreen)][ext_brew] + + +| Method | Command | Notes | +| - | - | - | +| pypi | `pipx install sherlock-project` | `pip` may be used in place of `pipx` | +| brew | `brew install sherlock` | Community supported | +| docker | `docker pull sherlock/sherlock` | | + +### Alternative guides and methods + +- [See all alternative guides][docs_install] +- [Python package][docs_py] +- [Docker container][docs_docker] -# change the working directory to sherlock -$ cd sherlock -# install the requirements -$ python3 -m pip install -r requirements.txt -``` ## Usage ```console -$ python3 sherlock --help +$ sherlock --help usage: sherlock [-h] [--version] [--verbose] [--folderoutput FOLDEROUTPUT] [--output OUTPUT] [--tor] [--unique-tor] [--csv] [--xlsx] [--site SITE_NAME] [--proxy PROXY_URL] [--json JSON_FILE] @@ -87,88 +93,28 @@ optional arguments: ``` To search for only one user: -``` -python3 sherlock user123 +```bash +sherlock user123 ``` To search for more than one user: -``` -python3 sherlock user1 user2 user3 +```bash +sherlock user1 user2 user3 ``` Accounts found will be stored in an individual text file with the corresponding username (e.g ```user123.txt```). -## Anaconda (Windows) Notes - -If you are using Anaconda in Windows, using `python3` might not work. Use `python` instead. - -## Docker Notes -If docker is installed you can build an image and run this as a container. +## How to contribute to Sherlock -``` -docker build -t mysherlock-image . -``` - -Once the image is built, sherlock can be invoked by running the following: - -``` -docker run --rm -t mysherlock-image user123 -``` - -Use the following command to access the saved results: - -``` -docker run --rm -t -v "$PWD/results:/opt/sherlock/results" mysherlock-image -o /opt/sherlock/results/text.txt user123 -``` - -Docker is instructed to create (or use) the folder `results` in the current working directory and to mount it at `/opt/sherlock/results` on the docker container by using the ```-v "$PWD/results:/opt/sherlock/results"``` options. `Sherlock` is instructed to export the result using the `-o /opt/sherlock/results/text.txt` option. - - -### Using `docker-compose` - -You can use the `docker-compose.yml` file from the repository and use this command: - -``` -docker-compose run sherlock -o /opt/sherlock/results/text.txt user123 -``` - -## Contributing We would love to have you help us with the development of Sherlock. Each and every contribution is greatly valued! Here are some things we would appreciate your help on: -- Addition of new site support ¹ -- Bringing back site support of [sites that have been removed](removed_sites.md) in the past due to false positives - -[1] Please look at the Wiki entry on [adding new sites](https://github.com/sherlock-project/sherlock/wiki/Adding-Sites-To-Sherlock) -to understand the issues. - -## Tests +- [Adding targets][docs_contrib_adding_targets] +- [Cleaning up existing targets][docs_contrib_removing_targets] +- [Restoring previously removed targets][docs_contrib_restoring_targets] -Thank you for contributing to Sherlock! - -Before creating a pull request with new development, please run the tests -to ensure that everything is working great. It would also be a good idea to run the tests -before starting development to distinguish problems between your -environment and the Sherlock software. - -The following is an example of the command line to run all the tests for -Sherlock. This invocation hides the progress text that Sherlock normally -outputs, and instead shows the verbose output of the tests. - -```console -$ cd sherlock/sherlock -$ python3 -m unittest tests.all --verbose -``` - -Note that we do currently have 100% test coverage. Unfortunately, some of -the sites that Sherlock checks are not always reliable, so it is common -to get response problems. Any problems in connection will show up as -warnings in the tests instead of true errors. - -If some sites are failing due to connection problems (site is down, in maintenance, etc) -you can exclude them from tests by creating a `tests/.excluded_sites` file with a -list of sites to ignore (one site name per line). +Head over to our __[Contributing][docs_contrib]__ page for additional info. ## Star History @@ -182,3 +128,20 @@ list of sites to ignore (one site name per line). MIT © Sherlock Project
Original Creator - [Siddharth Dushantha](https://github.com/sdushantha) + + + +[docs_install]: /docs/INSTALL.md +[docs_docker]: /docs/INSTALL.md#docker +[docs_docker_dockerhub]: /docs/INSTALL.md#docker +[docs_docker_compose]: /docs/INSTALL.md#using-compose +[docs_docker_source]: /docs/INSTALL.md#build-image-from-source-useful-for-contributors +[docs_py]: /docs/INSTALL.md#python +[docs_py_build]: /docs/INSTALL.md#build-python-package-from-source-useful-for-contributors +[docs_contrib]: /docs/CONTRIBUTING.md +[docs_contrib_adding_targets]: /docs/CONTRIBUTING.md#adding-targets +[docs_contrib_removing_targets]: /docs/CONTRIBUTING.md#removing-targets +[docs_contrib_restoring_targets]: /docs/CONTRIBUTING.md#restoring-targets +[ext_pypi]: https://pypi.org/project/sherlock-project/ +[ext_brew]: https://formulae.brew.sh/formula/sherlock + diff --git a/docs/pyproj/README.md b/docs/pyproj/README.md new file mode 100644 index 000000000..446382f5f --- /dev/null +++ b/docs/pyproj/README.md @@ -0,0 +1,46 @@ + + +

+
+ +
+ Hunt down social media accounts by username across social networks +
+ Additional documentation can be found on our GitHub repository +
+

+ +

+ + +

+ +## Usage + +```console +$ sherlock --help +usage: sherlock [-h] [--version] [--verbose] [--folderoutput FOLDEROUTPUT] + [--output OUTPUT] [--tor] [--unique-tor] [--csv] [--xlsx] + [--site SITE_NAME] [--proxy PROXY_URL] [--json JSON_FILE] + [--timeout TIMEOUT] [--print-all] [--print-found] [--no-color] + [--browse] [--local] [--nsfw] + USERNAMES [USERNAMES ...] +``` + +To search for only one user: +```bash +$ sherlock user123 +``` + +To search for more than one user: +```bash +$ sherlock user1 user2 user3 +``` + +## Star History + + + + + Sherlock Project Star History Chart + diff --git a/removed_sites.md b/docs/removed-sites.md similarity index 100% rename from removed_sites.md rename to docs/removed-sites.md diff --git a/sites.md b/docs/sites.md similarity index 100% rename from sites.md rename to docs/sites.md diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..7356c0d11 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project.scripts] +sherlock = "sherlock:main" + +[project.urls] +Homepage = "http://sherlock-project.github.io/" +Repository = "https://github.com/sherlock-project/sherlock.git" +Issues = "https://github.com/sherlock-project/sherlock/issues" + +[project] +name = "sherlock-project" +authors = [ + { name = "Siddharth Dushantha" } +] +maintainers = [ + { name = "Matheus Felipe" }, + { name = "Sondre Karlsen Dyrnes" }, + { name = "Paul Pfeister" } +] +description = "Hunt down social media accounts by username across social networks" +readme = "docs/pyproj/README.md" +# Do not set license to file. Causes issues with rpm packaging for some reason. +license = {text = "MIT"} +dynamic = ["dependencies", "version"] +keywords = [ "osint", "reconnaissance", "information gathering" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Topic :: Security" +] + +[tool.setuptools.dynamic] +dependencies = { file = [ "requirements.txt" ] } +version = { attr = "sherlock.__version__" } + +[tool.setuptools] +package-dir = {"" = "sherlock"} + +[tool.setuptools.package-data] +"*" = ["*.json"] diff --git a/removed_sites.json b/removed_sites.json deleted file mode 100644 index 1b107e553..000000000 --- a/removed_sites.json +++ /dev/null @@ -1,860 +0,0 @@ -{ - "AdobeForums": { - "errorType": "status_code", - "url": "https://forums.adobe.com/people/{}", - "urlMain": "https://forums.adobe.com/", - "username_claimed": "jack" - }, - "AngelList": { - "errorType": "status_code", - "url": "https://angel.co/u/{}", - "urlMain": "https://angel.co/", - "username_claimed": "blue" - }, - "Basecamp": { - "errorMsg": "The account you were looking for doesn't exist", - "errorType": "message", - "url": "https://{}.basecamphq.com", - "urlMain": "https://basecamp.com/", - "username_claimed": "blue" - }, - "BlackPlanet": { - "errorMsg": "My Hits", - "errorType": "message", - "url": "http://blackplanet.com/{}", - "urlMain": "http://blackplanet.com/" - }, - "Canva": { - "errorType": "response_url", - "errorUrl": "https://www.canva.com/{}", - "url": "https://www.canva.com/{}", - "urlMain": "https://www.canva.com/", - "username_claimed": "jenny" - }, - "Codementor": { - "errorType": "status_code", - "url": "https://www.codementor.io/@{}", - "urlMain": "https://www.codementor.io/", - "username_claimed": "blue" - }, - "EVE Online": { - "errorType": "response_url", - "errorUrl": "https://eveonline.com", - "url": "https://evewho.com/pilot/{}/", - "urlMain": "https://eveonline.com", - "username_claimed": "blue" - }, - "fanpop": { - "errorType": "response_url", - "errorUrl": "http://www.fanpop.com/", - "url": "http://www.fanpop.com/fans/{}", - "urlMain": "http://www.fanpop.com/", - "username_claimed": "blue" - }, - "Fotolog": { - "errorType": "status_code", - "url": "https://fotolog.com/{}", - "urlMain": "https://fotolog.com/" - }, - "Foursquare": { - "errorType": "status_code", - "url": "https://foursquare.com/{}", - "urlMain": "https://foursquare.com/", - "username_claimed": "dens" - }, - "gpodder.net": { - "errorType": "status_code", - "url": "https://gpodder.net/user/{}", - "urlMain": "https://gpodder.net/", - "username_claimed": "blue" - }, - "Investing.com": { - "errorType": "status_code", - "url": "https://www.investing.com/traders/{}", - "urlMain": "https://www.investing.com/", - "username_claimed": "jenny" - }, - "Khan Academy": { - "errorType": "status_code", - "url": "https://www.khanacademy.org/profile/{}", - "urlMain": "https://www.khanacademy.org/", - "username_claimed": "blue" - }, - "KiwiFarms": { - "errorMsg": "The specified member cannot be found", - "errorType": "message", - "url": "https://kiwifarms.net/members/?username={}", - "urlMain": "https://kiwifarms.net/", - "username_claimed": "blue" - }, - "NPM-Package": { - "errorType": "status_code", - "url": "https://www.npmjs.com/package/{}", - "urlMain": "https://www.npmjs.com/", - "username_claimed": "blue" - }, - "Pexels": { - "errorType": "status_code", - "url": "https://www.pexels.com/@{}", - "urlMain": "https://www.pexels.com/", - "username_claimed": "bruno" - }, - "Pixabay": { - "errorType": "status_code", - "url": "https://pixabay.com/en/users/{}", - "urlMain": "https://pixabay.com/", - "username_claimed": "blue" - }, - "PowerShell Gallery": { - "errorType": "status_code", - "url": "https://www.powershellgallery.com/profiles/{}", - "urlMain": "https://www.powershellgallery.com", - "username_claimed": "powershellteam" - }, - "RamblerDating": { - "errorType": "response_url", - "errorUrl": "https://dating.rambler.ru/page/{}", - "url": "https://dating.rambler.ru/page/{}", - "urlMain": "https://dating.rambler.ru/", - "username_claimed": "blue" - }, - "Shockwave": { - "errorMsg": "Oh no! You just finished all of the games on the internet!", - "errorType": "message", - "url": "http://www.shockwave.com/member/profiles/{}.jsp", - "urlMain": "http://www.shockwave.com/", - "username_claimed": "blue" - }, - "StreamMe": { - "errorType": "status_code", - "url": "https://www.stream.me/{}", - "urlMain": "https://www.stream.me/", - "username_claimed": "blue" - }, - "Teknik": { - "errorMsg": "The user does not exist", - "errorType": "message", - "url": "https://user.teknik.io/{}", - "urlMain": "https://teknik.io/", - "username_claimed": "red" - }, - "YandexMarket": { - "errorMsg": "\u0422\u0443\u0442 \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435\u0442", - "errorType": "message", - "url": "https://market.yandex.ru/user/{}/achievements", - "urlMain": "https://market.yandex.ru/", - "username_claimed": "blue" - }, - "Insanejournal": { - "errorMsg": "Unknown user", - "errorType": "message", - "url": "http://{}.insanejournal.com/profile", - "urlMain": "insanejournal.com", - "username_claimed": "blue" - }, - "Trip": { - "errorType": "status_code", - "url": "https://www.trip.skyscanner.com/user/{}", - "urlMain": "https://www.trip.skyscanner.com/", - "username_claimed": "blue" - }, - "SportsTracker": { - "errorUrl": "https://www.sports-tracker.com/page-not-found", - "errorType": "response_url", - "url": "https://www.sports-tracker.com/view_profile/{}", - "urlMain": "https://www.sports-tracker.com/", - "username_claimed": "blue" - }, - "boingboing.net": { - "errorType": "status_code", - "url": "https://bbs.boingboing.net/u/{}", - "urlMain": "https://boingboing.net/", - "username_claimed": "admin" - }, - "elwoRU": { - "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", - "errorType": "message", - "url": "https://elwo.ru/index/8-0-{}", - "urlMain": "https://elwo.ru/", - "username_claimed": "red" - }, - "ingvarr.net.ru": { - "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", - "errorType": "message", - "url": "http://ingvarr.net.ru/index/8-0-{}", - "urlMain": "http://ingvarr.net.ru/", - "username_claimed": "red" - }, - "Redsun.tf": { - "errorMsg": "The specified member cannot be found", - "errorType": "message", - "url": "https://forum.redsun.tf/members/?username={}", - "urlMain": "https://redsun.tf/", - "username_claimed": "dan" - }, - "CreativeMarket": { - "errorType": "status_code", - "url": "https://creativemarket.com/users/{}", - "urlMain": "https://creativemarket.com/", - "username_claimed": "blue" - }, - "pvpru": { - "errorType": "status_code", - "url": "https://pvpru.com/board/member.php?username={}&tab=aboutme#aboutme", - "urlMain": "https://pvpru.com/", - "username_claimed": "blue" - }, - "easyen": { - "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", - "errorType": "message", - "url": "https://easyen.ru/index/8-0-{}", - "urlMain": "https://easyen.ru/", - "username_claimed": "wd" - }, - "pedsovet": { - "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", - "errorType": "message", - "url": "http://pedsovet.su/index/8-0-{}", - "urlMain": "http://pedsovet.su/", - "username_claimed": "blue" - }, - "radioskot": { - "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", - "errorType": "message", - "url": "https://radioskot.ru/index/8-0-{}", - "urlMain": "https://radioskot.ru/", - "username_claimed": "red" - }, - "Coderwall": { - "errorMsg": "404! Our feels when that url is used", - "errorType": "message", - "url": "https://coderwall.com/{}", - "urlMain": "https://coderwall.com/", - "username_claimed": "jenny" - }, - "TamTam": { - "errorType": "response_url", - "errorUrl": "https://tamtam.chat/", - "url": "https://tamtam.chat/{}", - "urlMain": "https://tamtam.chat/", - "username_claimed": "blue" - }, - "Zomato": { - "errorType": "status_code", - "headers": { - "Accept-Language": "en-US,en;q=0.9" - }, - "url": "https://www.zomato.com/pl/{}/foodjourney", - "urlMain": "https://www.zomato.com/", - "username_claimed": "deepigoyal" - }, - "mixer.com": { - "errorType": "status_code", - "url": "https://mixer.com/{}", - "urlMain": "https://mixer.com/", - "urlProbe": "https://mixer.com/api/v1/channels/{}", - "username_claimed": "blue" - }, - "KanoWorld": { - "errorType": "status_code", - "url": "https://api.kano.me/progress/user/{}", - "urlMain": "https://world.kano.me/", - "username_claimed": "blue" - }, - "YandexCollection": { - "errorType": "status_code", - "url": "https://yandex.ru/collections/user/{}/", - "urlMain": "https://yandex.ru/collections/", - "username_claimed": "blue" - }, - "PayPal": { - "errorMsg": "", - "errorType": "message", - "url": "https://www.paypal.com/paypalme/{}", - "headers": { - "User-Agent": "" - }, - "urlMain": "https://www.paypal.me/", - "username_claimed": "blue" - }, - "ImageShack": { - "errorType": "response_url", - "errorUrl": "https://imageshack.us/", - "url": "https://imageshack.us/user/{}", - "urlMain": "https://imageshack.us/", - "username_claimed": "blue" - }, - "Aptoide": { - "errorType": "status_code", - "url": "https://{}.en.aptoide.com/", - "urlMain": "https://en.aptoide.com/", - "username_claimed": "blue" - }, - "Crunchyroll": { - "errorType": "status_code", - "url": "https://www.crunchyroll.com/user/{}", - "urlMain": "https://www.crunchyroll.com/", - "username_claimed": "blue" - }, - "T-MobileSupport": { - "errorType": "status_code", - "url": "https://support.t-mobile.com/people/{}", - "urlMain": "https://support.t-mobile.com", - "username_claimed": "blue" - }, - "OpenCollective": { - "errorType": "status_code", - "url": "https://opencollective.com/{}", - "urlMain": "https://opencollective.com/", - "username_claimed": "sindresorhus" - }, - "SegmentFault": { - "errorType": "status_code", - "url": "https://segmentfault.com/u/{}", - "urlMain": "https://segmentfault.com/", - "username_claimed": "bule" - }, - "Viadeo": { - "errorType": "status_code", - "url": "http://fr.viadeo.com/en/profile/{}", - "urlMain": "http://fr.viadeo.com/en/", - "username_claimed": "franck.patissier" - }, - "MeetMe": { - "errorType": "response_url", - "errorUrl": "https://www.meetme.com/", - "url": "https://www.meetme.com/{}", - "urlMain": "https://www.meetme.com/", - "username_claimed": "blue" - }, - "tracr.co": { - "errorMsg": "No search results", - "errorType": "message", - "regexCheck": "^[A-Za-z0-9]{2,32}$", - "url": "https://tracr.co/users/1/{}", - "urlMain": "https://tracr.co/", - "username_claimed": "blue" - }, - "Taringa": { - "errorType": "status_code", - "regexCheck": "^[^.]*$", - "url": "https://www.taringa.net/{}", - "urlMain": "https://taringa.net/", - "username_claimed": "blue" - }, - "Photobucket": { - "errorType": "status_code", - "url": "https://photobucket.com/user/{}/library", - "urlMain": "https://photobucket.com/", - "username_claimed": "blue" - }, - "4pda": { - "errorMsg": "[1,false,0]", - "errorType": "message", - "url": "https://4pda.ru/forum/index.php?act=search&source=pst&noform=1&username={}", - "urlMain": "https://4pda.ru/", - "urlProbe": " https://4pda.ru/forum/index.php?act=auth&action=chkname&login={}", - "username_claimed": "green" - }, - "PokerStrategy": { - "errorType": "status_code", - "url": "http://www.pokerstrategy.net/user/{}/profile/", - "urlMain": "http://www.pokerstrategy.net", - "username_claimed": "blue" - }, - "Filmogs": { - "errorType": "status_code", - "url": "https://www.filmo.gs/users/{}", - "urlMain": "https://www.filmo.gs/", - "username_claimed": "cupparober" - }, - "500px": { - "errorMsg": "No message available", - "errorType": "message", - "url": "https://500px.com/p/{}", - "urlMain": "https://500px.com/", - "urlProbe": "https://api.500px.com/graphql?operationName=ProfileRendererQuery&variables=%7B%22username%22%3A%22{}%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%224d02ff5c13927a3ac73b3eef306490508bc765956940c31051468cf30402a503%22%7D%7D", - "username_claimed": "blue" - }, - "Badoo": { - "errorType": "status_code", - "url": "https://badoo.com/profile/{}", - "urlMain": "https://badoo.com/", - "username_claimed": "blue" - }, - "Pling": { - "errorMsg": "Resource not found", - "errorType": "message", - "url": "https://www.pling.com/u/{}/", - "urlMain": "https://www.pling.com/", - "username_claimed": "blue" - }, - "Realmeye": { - "errorMsg": "Sorry, but we either:", - "errorType": "message", - "url": "https://www.realmeye.com/player/{}", - "urlMain": "https://www.realmeye.com/", - "username_claimed": "blue" - }, - "Travellerspoint": { - "errorMsg": "Wooops. Sorry!", - "errorType": "message", - "url": "https://www.travellerspoint.com/users/{}", - "urlMain": "https://www.travellerspoint.com", - "username_claimed": "blue" - }, - "GDProfiles": { - "errorType": "status_code", - "url": "https://gdprofiles.com/{}", - "urlMain": "https://gdprofiles.com/", - "username_claimed": "blue" - }, - "AllTrails": { - "errorMsg": "class=\"home index\"", - "errorType": "message", - "url": "https://www.alltrails.com/members/{}", - "urlMain": "https://www.alltrails.com/", - "username_claimed": "blue" - }, - "Cent": { - "errorMsg": "Cent", - "errorType": "message", - "url": "https://beta.cent.co/@{}", - "urlMain": "https://cent.co/", - "username_claimed": "blue" - }, - "Anobii": { - "errorType": "response_url", - "url": "https://www.anobii.com/{}/profile", - "urlMain": "https://www.anobii.com/", - "username_claimed": "blue" - }, - "Kali community": { - "errorMsg": "This user has not registered and therefore does not have a profile to view.", - "errorType": "message", - "url": "https://forums.kali.org/member.php?username={}", - "urlMain": "https://forums.kali.org/", - "username_claimed": "blue" - }, - "NameMC (Minecraft.net skins)": { - "errorMsg": "Profiles: 0 results", - "errorType": "message", - "url": "https://namemc.com/profile/{}", - "urlMain": "https://namemc.com/", - "username_claimed": "blue" - }, - "Steamid": { - "errorMsg": "", - "errorType": "message", - "url": "https://steamid.uk/profile/{}", - "urlMain": "https://steamid.uk/", - "username_claimed": "blue" - }, - "TripAdvisor": { - "errorMsg": "This page is on vacation\u2026", - "errorType": "message", - "url": "https://tripadvisor.com/members/{}", - "urlMain": "https://tripadvisor.com/", - "username_claimed": "blue" - }, - "House-Mixes.com": { - "errorMsg": "Profile Not Found", - "errorType": "message", - "regexCheck": "^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*$", - "url": "https://www.house-mixes.com/profile/{}", - "urlMain": "https://www.house-mixes.com/", - "username_claimed": "blue" - }, - "Quora": { - "errorMsg": "Page Not Found", - "errorType": "message", - "url": "https://www.quora.com/profile/{}", - "urlMain": "https://www.quora.com/", - "username_claimed": "Matt-Riggsby" - }, - "SparkPeople": { - "errorMsg": "We couldn't find that user", - "errorType": "message", - "url": "https://www.sparkpeople.com/mypage.asp?id={}", - "urlMain": "https://www.sparkpeople.com", - "username_claimed": "adam" - }, - "Cloob": { - "errorType": "status_code", - "url": "https://www.cloob.com/name/{}", - "urlMain": "https://www.cloob.com/", - "username_claimed": "blue" - }, - "TM-Ladder": { - "errorMsg": "player unknown or invalid", - "errorType": "message", - "url": "http://en.tm-ladder.com/{}_rech.php", - "urlMain": "http://en.tm-ladder.com/index.php", - "username_claimed": "blue" - }, - "plug.dj": { - "errorType": "status_code", - "url": "https://plug.dj/@/{}", - "urlMain": "https://plug.dj/", - "username_claimed": "plug-dj-rock" - }, - "Facenama": { - "errorType": "response_url", - "errorUrl": "https://facenama.com/404.html", - "regexCheck": "^[-a-zA-Z0-9_]+$", - "url": "https://facenama.com/{}", - "urlMain": "https://facenama.com/", - "username_claimed": "blue" - }, - "Designspiration": { - "errorType": "status_code", - "url": "https://www.designspiration.net/{}/", - "urlMain": "https://www.designspiration.net/", - "username_claimed": "blue" - }, - "CapFriendly": { - "errorMsg": "
No results found
", - "errorType": "message", - "regexCheck": "^[a-zA-z][a-zA-Z0-9_]{2,79}$", - "url": "https://www.capfriendly.com/users/{}", - "urlMain": "https://www.capfriendly.com/", - "username_claimed": "thisactuallyexists" - }, - "Gab": { - "errorMsg": "The page you are looking for isn't here.", - "errorType": "message", - "url": "https://gab.com/{}", - "urlMain": "https://gab.com", - "username_claimed": "a" - }, - "FanCentro": { - "errorMsg": "var environment", - "errorType": "message", - "url": "https://fancentro.com/{}", - "urlMain": "https://fancentro.com/", - "username_claimed": "nielsrosanna" - }, - "Codeforces": { - "errorType": "response_url", - "errorUrl": "https://codeforces.com/", - "url": "https://codeforces.com/profile/{}", - "urlMain": "https://www.codeforces.com/", - "username_claimed": "tourist" - }, - "Smashcast": { - "errorType": "status_code", - "url": "https://www.smashcast.tv/api/media/live/{}", - "urlMain": "https://www.smashcast.tv/", - "username_claimed": "hello" - }, - "Countable": { - "errorType": "status_code", - "url": "https://www.countable.us/{}", - "urlMain": "https://www.countable.us/", - "username_claimed": "blue" - }, - "Spotify": { - "errorType": "status_code", - "url": "https://open.spotify.com/user/{}", - "urlMain": "https://open.spotify.com/", - "username_claimed": "blue" - }, - "Raidforums": { - "errorType": "status_code", - "url": "https://raidforums.com/User-{}", - "urlMain": "https://raidforums.com/", - "username_claimed": "red" - }, - "Pinterest": { - "errorType": "status_code", - "url": "https://www.pinterest.com/{}/", - "urlMain": "https://www.pinterest.com/", - "username_claimed": "blue" - }, - "PCPartPicker": { - "errorType": "status_code", - "url": "https://pcpartpicker.com/user/{}", - "urlMain": "https://pcpartpicker.com", - "username_claimed": "blue" - }, - "eBay.com": { - "errorMsg": "The User ID you entered was not found. Please check the User ID and try again.", - "errorType": "message", - "url": "https://www.ebay.com/usr/{}", - "urlMain": "https://www.ebay.com/", - "username_claimed": "blue" - }, - "eBay.de": { - "errorMsg": "Der eingegebene Nutzername wurde nicht gefunden. Bitte pr\u00fcfen Sie den Nutzernamen und versuchen Sie es erneut.", - "errorType": "message", - "url": "https://www.ebay.de/usr/{}", - "urlMain": "https://www.ebay.de/", - "username_claimed": "blue" - }, - "Ghost": { - "errorMsg": "Domain Error", - "errorType": "message", - "url": "https://{}.ghost.io/", - "urlMain": "https://ghost.org/", - "username_claimed": "troyhunt" - }, - "Atom Discussions": { - "errorMsg": "Oops! That page doesn\u2019t exist or is private.", - "errorType": "message", - "url": "https://discuss.atom.io/u/{}/summary", - "urlMain": "https://discuss.atom.io", - "username_claimed": "blue" - }, - "Gam1ng": { - "errorType": "status_code", - "url": "https://gam1ng.com.br/user/{}", - "urlMain": "https://gam1ng.com.br", - "username_claimed": "PinKgirl" - }, - "OGUsers": { - "errorType": "status_code", - "url": "https://ogusers.com/{}", - "urlMain": "https://ogusers.com/", - "username_claimed": "ogusers" - }, - "Otzovik": { - "errorType": "status_code", - "url": "https://otzovik.com/profile/{}", - "urlMain": "https://otzovik.com/", - "username_claimed": "blue" - }, - "radio_echo_msk": { - "errorType": "status_code", - "url": "https://echo.msk.ru/users/{}", - "urlMain": "https://echo.msk.ru/", - "username_claimed": "blue" - }, - "Ello": { - "errorMsg": "We couldn't find the page you're looking for", - "errorType": "message", - "url": "https://ello.co/{}", - "urlMain": "https://ello.co/", - "username_claimed": "blue" - }, - "GitHub Support Community": { - "errorMsg": "Oops! That page doesn\u2019t exist or is private.", - "errorType": "message", - "url": "https://github.community/u/{}/summary", - "urlMain": "https://github.community", - "username_claimed": "jperl" - }, - "GuruShots": { - "errorType": "status_code", - "url": "https://gurushots.com/{}/photos", - "urlMain": "https://gurushots.com/", - "username_claimed": "blue" - }, - "Google Developer": { - "errorMsg": "Sorry, the profile was not found.", - "errorType": "message", - "url": "https://g.dev/{}", - "urlMain": "https://g.dev/", - "username_claimed": "blue" - }, - "mastodon.technology": { - "errorType": "status_code", - "url": "https://mastodon.technology/@{}", - "urlMain": "https://mastodon.xyz/", - "username_claimed": "ashfurrow" - }, - "zoomit": { - "errorMsg": "\u0645\u062a\u0627\u0633\u0641\u0627\u0646\u0647 \u0635\u0641\u062d\u0647 \u06cc\u0627\u0641\u062a \u0646\u0634\u062f", - "errorType": "message", - "url": "https://www.zoomit.ir/user/{}", - "urlMain": "https://www.zoomit.ir", - "username_claimed": "kossher" - }, - "Facebook": { - "errorType": "status_code", - "regexCheck": "^[a-zA-Z0-9\\.]{3,49}(?Tinder | Dating, Make Friends & Meet New People", - "Tinder | Match. Chat. Date." - ], - "errorType": "message", - "url": "https://www.tinder.com/@{}", - "urlMain": "https://tinder.com/", - "username_claimed": "blue" - }, - "Coil": { - "errorMsg": "User not found", - "errorType": "message", - "request_method": "POST", - "request_payload": { - "operationName": "getCreator", - "query": "query getCreator($userShortName:String!){getCreator(userShortName:$userShortName){id}}", - "variables": { - "userShortName": "{}" - } - }, - "url": "https://coil.com/u/{}", - "urlMain": "https://coil.com/", - "urlProbe": "https://coil.com/gateway", - "username_claimed": "adam" - }, - "OnlyFans": { - "errorType": "status_code", - "isNSFW": true, - "url": "https://onlyfans.com/{}", - "urlMain": "https://onlyfans.com/", - "urlProbe": "https://onlyfans.com/api2/v2/users/{}", - "username_claimed": "theemilylynne" - }, - "OK": { - "errorType": "status_code", - "regexCheck": "^[a-zA-Z][a-zA-Z0-9_.-]*$", - "url": "https://ok.ru/{}", - "urlMain": "https://ok.ru/", - "username_claimed": "ok" - }, - "forumhouseRU": { - "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f.", - "errorType": "message", - "url": "https://www.forumhouse.ru/members/?username={}", - "urlMain": "https://www.forumhouse.ru/", - "username_claimed": "red" - }, - "Enjin": { - "errorMsg": "Yikes, there seems to have been an error. We've taken note and will check out the problem right away!", - "errorType": "message", - "url": "https://www.enjin.com/profile/{}", - "urlMain": "https://www.enjin.com/", - "username_claimed": "blue" - }, - "IRL": { - "errorType": "status_code", - "url": "https://www.irl.com/{}", - "urlMain": "https://www.irl.com/", - "username_claimed": "hacker" - }, - "Munzee": { - "errorType": "status_code", - "url": "https://www.munzee.com/m/{}", - "urlMain": "https://www.munzee.com/", - "username_claimed": "blue" - }, - "Quizlet": { - "errorMsg": "Page Unavailable", - "errorType": "message", - "url": "https://quizlet.com/{}", - "urlMain": "https://quizlet.com", - "username_claimed": "blue" - }, - "GunsAndAmmo": { - "errorType": "status_code", - "url": "https://forums.gunsandammo.com/profile/{}", - "urlMain": "https://gunsandammo.com/", - "username_claimed": "adam" - }, - "TikTok": { - "errorType": "status_code", - "url": "https://tiktok.com/@{}", - "urlMain": "https://tiktok.com/", - "username_claimed": "red" - }, - "Lolchess": { - "errorMsg": "No search results", - "errorType": "message", - "url": "https://lolchess.gg/profile/na/{}", - "urlMain": "https://lolchess.gg/", - "username_claimed": "blue" - }, - "Virgool": { - "errorMsg": "\u06f4\u06f0\u06f4", - "errorType": "message", - "url": "https://virgool.io/@{}", - "urlMain": "https://virgool.io/", - "username_claimed": "blue" - }, - "Whonix Forum": { - "errorType": "status_code", - "url": "https://forums.whonix.org/u/{}/summary", - "urlMain": "https://forums.whonix.org/", - "username_claimed": "red" - }, - "ebio.gg": { - "errorType": "status_code", - "url": "https://ebio.gg/{}", - "urlMain": "https:/ebio.gg", - "username_claimed": "dev" - }, - "metacritic": { - "errorMsg": "User not found", - "errorType": "message", - "regexCheck": "^(?![-_].)[A-Za-z0-9-_]{3,15}$", - "url": "https://www.metacritic.com/user/{}", - "urlMain": "https://www.metacritic.com/", - "username_claimed": "blue" - }, - "Oracle Communities": { - "errorType": "status_code", - "url": "https://community.oracle.com/people/{}", - "urlMain": "https://community.oracle.com", - "username_claimed": "dev" - }, - "HexRPG": { - "errorMsg": "Error : User ", - "errorType": "message", - "regexCheck": "^[a-zA-Z0-9_ ]{3,20}$", - "url": "https://www.hexrpg.com/userinfo/{}", - "urlMain": "https://www.hexrpg.com/", - "username_claimed": "blue" - }, - "G2G": { - "errorType": "response_url", - "errorUrl": "https://www.g2g.com/{}", - "regexCheck": "^[A-Za-z][A-Za-z0-9_]{2,11}$", - "url": "https://www.g2g.com/{}", - "urlMain": "https://www.g2g.com/", - "username_claimed": "user" - }, - "BitCoinForum": { - "errorMsg": "The user whose profile you are trying to view does not exist.", - "errorType": "message", - "url": "https://bitcoinforum.com/profile/{}", - "urlMain": "https://bitcoinforum.com", - "username_claimed": "bitcoinforum.com" - } -} diff --git a/requirements.txt b/requirements.txt index 0bee5bfdf..b1dd62522 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ requests-futures>=1.0.0 stem>=1.8.0 torrequest>=0.1.0 pandas>=1.0.0 -openpyxl<=3.0.10 -exrex>=0.11.0 \ No newline at end of file +openpyxl>=3.0.10 +exrex>=0.11.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..8929bbae7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = Sherlock +author = Sherlock Project +url = http://sherlock-project.github.io/ diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..434d3f95e --- /dev/null +++ b/setup.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +"""Sometimes required for rpm spec - particularly when using pyp2rpm +""" + +from setuptools import setup + +setup() diff --git a/sherlock-project.spec b/sherlock-project.spec new file mode 100644 index 000000000..b07dded1a --- /dev/null +++ b/sherlock-project.spec @@ -0,0 +1,71 @@ +# Packager: Paul Pfeister (GitHub @ppfeister) +%global source_ref master + +Name: sherlock-project +Version: 0.14.4 +Release: %autorelease +Summary: Hunt down social media accounts by username across social networks + +License: MIT +URL: https://github.com/sherlock-project/sherlock +Source: %{url}/archive/%{source_ref}.tar.gz +# Switch to new Source URL after adoption of tagged releases + +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: help2man + +%global _description %{expand: +Hunt down social media accounts by username across 400+ social networks and +websites. New targets are tested and implemented regularly. +} + +%description %{_description} + + +%prep +%autosetup -n sherlock-%{source_ref} + + +%generate_buildrequires +%pyproject_buildrequires + + +%build +%pyproject_wheel + + +%install +%pyproject_install +%pyproject_save_files -l sherlock sites result notify __init__ __main__ + +sed -r -i '1{/^#!/d}' '%{buildroot}%{python3_sitelib}/__main__.py' +sed -r -i '1{/^#!/d}' '%{buildroot}%{python3_sitelib}/sherlock.py' + +install -d '%{buildroot}%{_mandir}/man1' +PYTHONPATH='%{buildroot}%{python3_sitelib}' help2man \ + --no-info \ + --version-string='%{version}' \ + --name='%{summary}' \ + --output='%{buildroot}%{_mandir}/man1/sherlock.1' \ + '%{buildroot}%{_bindir}/sherlock' + + +%check +# Tests fail when pwd isn't sherlock. Relative pathing need fix upstream. +cd sherlock +%{py3_test_envvars} %{python3} -m unittest tests.all --verbose +cd .. + + +%files -f %{pyproject_files} +%doc README.md +%{_bindir}/sherlock +%{python3_sitelib}/resources +%pycached %{python3_sitelib}/tests/*.py +%{_mandir}/man1/sherlock.1* + + +%changelog +* Tue May 14 2024 Paul Pfeister 0.14.4-1 +- Initial package. diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 6414fdbe8..261c16749 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -161,7 +161,11 @@ "username_claimed": "test" }, "Archive.org": { - "errorMsg": "could not fetch an account with user item identifier", + "__comment__": "'The resource could not be found' relates to archive downtime", + "errorMsg": [ + "could not fetch an account with user item identifier", + "The resource could not be found" + ], "errorType": "message", "url": "https://archive.org/details/@{}", "urlMain": "https://archive.org", @@ -713,7 +717,8 @@ "username_claimed": "jonasjacobsson" }, "EyeEm": { - "errorType": "status_code", + "errorMsg": "Whoops! We can't find the page you're looking for...", + "errorType": "message", "url": "https://www.eyeem.com/u/{}", "urlMain": "https://www.eyeem.com/", "username_claimed": "blue" @@ -1213,6 +1218,7 @@ "username_claimed": "blue" }, "Kick": { + "__comment__": "Cloudflare. Only viable when proxied.", "errorMsg": "Not Found", "errorType": "message", "url": "https://kick.com/{}", diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index bd19b43b5..cbd258667 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -143,7 +143,6 @@ def check_for_parameter(username): return "{?}" in username -checksymbols = [] checksymbols = ["_", "-", "."] @@ -200,7 +199,12 @@ def sherlock( # Create session based on request methodology if tor or unique_tor: # Requests using Tor obfuscation - underlying_request = TorRequest() + try: + underlying_request = TorRequest() + except OSError: + print("Tor not found in system path. Unable to continue.\n") + sys.exit(query_notify.finish()) + underlying_session = underlying_request.session else: # Normal requests @@ -377,11 +381,13 @@ def sherlock( query_status = QueryStatus.UNKNOWN error_context = None - # As WAFs advance and evolve, they will occasionally block Sherlock and lead to false positives - # and negatives. Fingerprints should be added here to filter results that fail to bypass WAFs. - # Fingerprints should be highly targetted. Comment at the end of each fingerprint to indicate target and date. + # As WAFs advance and evolve, they will occasionally block Sherlock and + # lead to false positives and negatives. Fingerprints should be added + # here to filter results that fail to bypass WAFs. Fingerprints should + # be highly targetted. Comment at the end of each fingerprint to + # indicate target and date fingerprinted. WAFHitMsgs = [ - '.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark .big-button,body.dark .pow-button{background-color:#4693ff;color:#1d1d1d}body.dark #challenge-success-text{background-image:url(data:image/svg+xml;base64,', # 2024-04-08 Cloudflare + '.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark', # 2024-05-13 Cloudflare '{return l.onPageView}}),Object.defineProperty(r,"perimeterxIdentifiers",{enumerable:' # 2024-04-09 PerimeterX / Human Security ]