-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from dapper91/dev
initial version.
- Loading branch information
Showing
27 changed files
with
3,585 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
[flake8] | ||
max-line-length = 120 | ||
per-file-ignores = | ||
generic_connection_pool/*__init__.py: F401 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
name: release | ||
|
||
on: | ||
release: | ||
types: | ||
- released | ||
|
||
jobs: | ||
release: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v2 | ||
- name: Set up Python | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: '3.x' | ||
- name: Install dependencies | ||
run: | | ||
python -m pip install --upgrade pip poetry | ||
poetry install | ||
- name: Build and publish | ||
run: | | ||
poetry build | ||
poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
name: test | ||
|
||
on: | ||
pull_request: | ||
branches: | ||
- dev | ||
- master | ||
push: | ||
branches: | ||
- master | ||
|
||
jobs: | ||
test: | ||
runs-on: ubuntu-latest | ||
strategy: | ||
matrix: | ||
python-version: ['3.9', '3.10', '3.11'] | ||
steps: | ||
- uses: actions/checkout@v2 | ||
- name: Set up Python ${{ matrix.python-version }} | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
- name: Install dependencies | ||
run: | | ||
python -m pip install --upgrade pip | ||
pip install poetry | ||
poetry install --no-root | ||
- name: Run pre-commit hooks | ||
run: poetry run pre-commit run --hook-stage merge-commit --all-files | ||
- name: Run tests | ||
run: PYTHONPATH="$(pwd):$PYTHONPATH" poetry run py.test --cov=generic_connection_pool --cov-report=xml tests | ||
- name: Upload coverage to Codecov | ||
uses: codecov/codecov-action@v1 | ||
with: | ||
token: ${{ secrets.CODECOV_TOKEN }} | ||
files: ./coverage.xml | ||
flags: unittests | ||
fail_ci_if_error: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -127,3 +127,6 @@ dmypy.json | |
|
||
# Pyre type checker | ||
.pyre/ | ||
|
||
# poetry | ||
poetry.lock |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
default_stages: | ||
- commit | ||
- merge-commit | ||
|
||
repos: | ||
- repo: https://github.com/pre-commit/pre-commit-hooks | ||
rev: v4.4.0 | ||
hooks: | ||
- id: check-yaml | ||
- id: check-toml | ||
- id: trailing-whitespace | ||
- id: end-of-file-fixer | ||
stages: | ||
- commit | ||
- id: mixed-line-ending | ||
name: fix line ending | ||
stages: | ||
- commit | ||
args: | ||
- --fix=lf | ||
- id: mixed-line-ending | ||
name: check line ending | ||
stages: | ||
- merge-commit | ||
args: | ||
- --fix=no | ||
- repo: https://github.com/asottile/add-trailing-comma | ||
rev: v2.4.0 | ||
hooks: | ||
- id: add-trailing-comma | ||
stages: | ||
- commit | ||
- repo: https://github.com/pre-commit/mirrors-autopep8 | ||
rev: v2.0.2 | ||
hooks: | ||
- id: autopep8 | ||
stages: | ||
- commit | ||
args: | ||
- --diff | ||
- repo: https://github.com/pycqa/flake8 | ||
rev: 6.0.0 | ||
hooks: | ||
- id: flake8 | ||
- repo: https://github.com/pycqa/isort | ||
rev: 5.12.0 | ||
hooks: | ||
- id: isort | ||
name: fix import order | ||
stages: | ||
- commit | ||
args: | ||
- --line-length=120 | ||
- --multi-line=9 | ||
- --project=generic_connection_pool | ||
- id: isort | ||
name: check import order | ||
stages: | ||
- merge-commit | ||
args: | ||
- --check-only | ||
- --line-length=120 | ||
- --multi-line=9 | ||
- --project=generic_connection_pool | ||
- repo: https://github.com/pre-commit/mirrors-mypy | ||
rev: v1.1.1 | ||
hooks: | ||
- id: mypy | ||
stages: | ||
- commit | ||
name: mypy | ||
pass_filenames: false | ||
additional_dependencies: | ||
- types-psycopg2 | ||
args: ["--package", "generic_connection_pool"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
Changelog | ||
========= | ||
|
||
|
||
0.1.0 (2021-03-15) | ||
------------------ | ||
|
||
- Initial release |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,234 @@ | ||
======================= | ||
generic-connection-pool | ||
======================= | ||
|
||
.. image:: https://static.pepy.tech/personalized-badge/generic-connection-pool?period=month&units=international_system&left_color=grey&right_color=orange&left_text=Downloads/month | ||
:target: https://pepy.tech/project/generic-connection-pool | ||
:alt: Downloads/month | ||
.. image:: https://github.com/dapper91/generic-connection-pool/actions/workflows/test.yml/badge.svg?branch=master | ||
:target: https://github.com/dapper91/generic-connection-pool/actions/workflows/test.yml | ||
:alt: Build status | ||
.. image:: https://img.shields.io/pypi/l/generic-connection-pool.svg | ||
:target: https://pypi.org/project/generic-connection-pool | ||
:alt: License | ||
.. image:: https://img.shields.io/pypi/pyversions/generic-connection-pool.svg | ||
:target: https://pypi.org/project/generic-connection-pool | ||
:alt: Supported Python versions | ||
.. image:: https://codecov.io/gh/dapper91/generic-connection-pool/branch/master/graph/badge.svg | ||
:target: https://codecov.io/gh/dapper91/generic-connection-pool | ||
:alt: Code coverage | ||
|
||
|
||
``generic-connection-pool`` is a connection pool that can be used for TCP, http, database connections. | ||
|
||
Features: | ||
|
||
- **generic nature**: can be used for any connection you desire (TCP, http, database) | ||
- **runtime agnostic**: synchronous and asynchronous pool supported | ||
- **flexibility**: flexable connection retention and recycling policy | ||
- **fully-typed**: mypy type-checker compatible | ||
|
||
|
||
Installation | ||
------------ | ||
|
||
You can install generic-connection-pool with pip: | ||
|
||
.. code-block:: console | ||
$ pip install generic-connection-pool | ||
Quickstart | ||
---------- | ||
|
||
The following example illustrates how to create asynchronous ssl socket pool: | ||
|
||
.. code-block:: python | ||
import asyncio | ||
from typing import Tuple | ||
from generic_connection_pool.asyncio import ConnectionPool | ||
from generic_connection_pool.contrib.socket_async import TcpStreamConnectionManager | ||
Hostname = str | ||
Port = int | ||
Endpoint = Tuple[Hostname, Port] | ||
Connection = Tuple[asyncio.StreamReader, asyncio.StreamWriter] | ||
async def main() -> None: | ||
pool = ConnectionPool[Endpoint, Connection]( | ||
TcpStreamConnectionManager(ssl=True), | ||
idle_timeout=30.0, | ||
max_lifetime=600.0, | ||
min_idle=3, | ||
max_size=20, | ||
total_max_size=100, | ||
background_collector=True, | ||
) | ||
async with pool.connection(endpoint=('www.wikipedia.org', 443), timeout=5.0) as (reader, writer): | ||
request = ( | ||
'GET / HTTP/1.0\n' | ||
'Host: www.wikipedia.org\n' | ||
'\n' | ||
'\n' | ||
) | ||
writer.write(request.encode()) | ||
await writer.drain() | ||
response = await reader.read() | ||
print(response.decode()) | ||
asyncio.run(main()) | ||
Configuration | ||
------------- | ||
|
||
Synchronous and asynchronous pools supports the following parameters: | ||
|
||
* **connection_manager**: connection manager instance | ||
* **acquire_timeout**: connection acquiring default timeout | ||
* **dispose_batch_size**: number of connections to be disposed at once | ||
(if background collector is started the parameter is ignored) | ||
* **dispose_timeout**: connection disposal timeout | ||
* **background_collector**: start worker that disposes timed-out connections in background maintain provided pool state | ||
otherwise they will be disposed on each connection release | ||
* **idle_timeout**: number of seconds after which a connection will be closed respecting min_idle parameter | ||
(the connection will be closed only if the connection number exceeds min_idle) | ||
* **max_lifetime**: number of seconds after which a connection will be closed (min_idle parameter will be ignored) | ||
* **min_idle**: minimum number of connections the pool tries to hold (for each endpoint) | ||
* **max_size**: maximum number of connections (for each endpoint) | ||
* **total_max_size**: maximum number of connections (for all endpoints) | ||
|
||
|
||
Generic nature | ||
-------------- | ||
|
||
Since the pool has generic nature is can be used for database connections as well: | ||
|
||
.. code-block:: python | ||
import psycopg2.extensions | ||
from generic_connection_pool.contrib.psycopg2 import DbConnectionManager | ||
from generic_connection_pool.threding import ConnectionPool | ||
Endpoint = str | ||
Connection = psycopg2.extensions.connection | ||
def main() -> None: | ||
dsn_params = dict(dbname='postgres', user='postgres', password='secret') | ||
pool = ConnectionPool[Endpoint, Connection]( | ||
DbConnectionManager( | ||
dsn_params={ | ||
'master': dict(dsn_params, host='db-master.local'), | ||
'replica-1': dict(dsn_params, host='db-replica-1.local'), | ||
'replica-2': dict(dsn_params, host='db-replica-2.local'), | ||
}, | ||
), | ||
acquire_timeout=2.0, | ||
idle_timeout=60.0, | ||
max_lifetime=600.0, | ||
min_idle=3, | ||
max_size=10, | ||
total_max_size=15, | ||
background_collector=True, | ||
) | ||
with pool.connection(endpoint='master') as conn: | ||
cur = conn.cursor() | ||
cur.execute("SELECT * FROM pg_stats;") | ||
print(cur.fetchone()) | ||
with pool.connection(endpoint='replica-1') as conn: | ||
cur = conn.cursor() | ||
cur.execute("SELECT * FROM pg_stats;") | ||
print(cur.fetchone()) | ||
pool.close() | ||
main() | ||
Extendability | ||
------------- | ||
|
||
If built-in connection managers are not suitable for your task the one can be easily created by yourself: | ||
|
||
.. code-block:: python | ||
import socket | ||
from ssl import SSLContext, SSLSocket | ||
from typing import Optional, Tuple | ||
from generic_connection_pool.threding import BaseConnectionManager, ConnectionPool | ||
Hostname = str | ||
Port = int | ||
SslEndpoint = Tuple[Hostname, Port] | ||
Connection = SSLSocket | ||
class SslSocketConnectionManager(BaseConnectionManager[SslEndpoint, Connection]): | ||
""" | ||
SSL socket connection manager. | ||
""" | ||
def __init__(self, ssl: SSLContext): | ||
self._ssl = ssl | ||
def create(self, endpoint: SslEndpoint, timeout: Optional[float] = None) -> Connection: | ||
hostname, port = endpoint | ||
sock = self._ssl.wrap_socket(socket.socket(type=socket.SOCK_STREAM), server_hostname=hostname) | ||
sock.settimeout(timeout) | ||
sock.connect((hostname, port)) | ||
return sock | ||
def dispose(self, endpoint: SslEndpoint, conn: Connection, timeout: Optional[float] = None) -> None: | ||
conn.settimeout(timeout) | ||
try: | ||
conn.shutdown(socket.SHUT_RDWR) | ||
except OSError: | ||
pass | ||
conn.close() | ||
def main() -> None: | ||
pool = ConnectionPool[SslEndpoint, Connection]( | ||
SslSocketConnectionManager(ssl=SSLContext()), | ||
idle_timeout=30.0, | ||
max_lifetime=600.0, | ||
min_idle=3, | ||
max_size=20, | ||
total_max_size=100, | ||
background_collector=True, | ||
) | ||
with pool.connection(endpoint=('www.wikipedia.org', 443), timeout=5.0) as sock: | ||
request = ( | ||
'GET / HTTP/1.0\n' | ||
'Host: www.wikipedia.org\n' | ||
'\n' | ||
'\n' | ||
) | ||
sock.write(request.encode()) | ||
response = [] | ||
while chunk := sock.recv(): | ||
response.append(chunk) | ||
print(b''.join(response).decode()) | ||
pool.close() | ||
main() |
Oops, something went wrong.