Merge branch 'main' into 22-export-configuration-changes

app/main.py

@@ -4,15 +4,18 @@
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.templating import Jinja2Templates
+from fastapi.responses import JSONResponse
 from fastapi_cache import caches, close_caches
 from fastapi_cache.backends.redis import CACHE_KEY, RedisCacheBackend
 from tableauhyperapi import HyperProcess, Telemetry
 from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
 from starlette.middleware.sessions import SessionMiddleware
+from starlette_exporter import PrometheusMiddleware, handle_metrics
+from redis import Redis
 
 from app.common_tags import HYPER_PROCESS_CACHE_KEY
-from app.database import engine
-from app.models import Base
+from app.database import engine, SessionLocal
+from app.models import Base, HyperFile
 from app.settings import settings
 from app.utils.onadata_utils import schedule_all_active_forms
 from app.routers.file import router as file_router
@@ -46,11 +49,15 @@
     allow_headers=settings.cors_allowed_headers,
     max_age=settings.cors_max_age,
 )
+app.add_middleware(
+    PrometheusMiddleware, app_name="duva", prefix="duva", filter_unhandled_paths=True
+)
 if settings.sentry_dsn:
     sentry_sdk.init(dsn=settings.sentry_dsn, release=settings.app_version)
     app.add_middleware(SentryAsgiMiddleware)
 
 # Include routes
+app.add_route("/metrics", handle_metrics)
 app.include_router(server_router, tags=["Server Configuration"])
 app.include_router(oauth_router, tags=["OAuth2"])
 app.include_router(file_router, tags=["Hyper File"])
@@ -67,6 +74,36 @@ def home(request: Request):
     }
 
 
+@app.get("/health", tags=["Application"])
+def service_health(request: Request):
+    db = SessionLocal()
+    database_reachable = True
+    cache_reachable = True
+
+    try:
+        HyperFile.get_active_files(db)
+    except Exception:
+        database_reachable = False
+
+    db.close()
+
+    redis_client = Redis(
+        host=settings.redis_host, port=settings.redis_port, db=settings.redis_db
+    )
+    try:
+        redis_client.ping()
+    except Exception:
+        cache_reachable = False
+
+    return JSONResponse(
+        {
+            "Database": "OK" if database_reachable else "FAILING",
+            "Cache": "OK" if cache_reachable else "FAILING",
+        },
+        200 if (database_reachable and cache_reachable) else 500,
+    )
+
+
 @app.on_event("startup")
 async def on_startup() -> None:
     # Ensure media file path exists

app/utils/onadata_utils.py

@@ -11,6 +11,7 @@
 from fastapi_cache import caches
 from sqlalchemy.orm.session import Session
 from tableauhyperapi import HyperProcess, Telemetry
+from prometheus_client import Gauge, Counter
 
 from app import schemas
 from app.common_tags import (
@@ -30,6 +31,19 @@
 )
 
 
+IN_PROGRESS_HYPER_IMPORT = Gauge(
+    "in_progress_hyper_import",
+    "Number of Import processes currently running for Tableau Hyper databases",
+)
+SUCCESSFUL_IMPORTS = Counter(
+    "successful_hyper_database_imports",
+    "Number of successfull imports to a hyper database",
+)
+FAILED_IMPORTS = Counter(
+    "failed_hyper_database_imports", "Number of failed imports to a hyper database"
+)
+
+
 class UnsupportedForm(Exception):
     pass
 
@@ -154,6 +168,7 @@ def get_csv_export(
                 return Path(csv_export.name)
 
 
+@IN_PROGRESS_HYPER_IMPORT.track_inprogress()
 def start_csv_import_to_hyper(
     hyperfile_id: int,
     process: HyperProcess,
@@ -221,7 +236,9 @@ def start_csv_import_to_hyper(
                 )
                 db.close()
                 if err:
+                    FAILED_IMPORTS.inc()
                     sentry_sdk.capture_exception(err)
+                SUCCESSFUL_IMPORTS.inc()
                 return successful_import
         except LockError:
             pass

requirements.in

@@ -16,3 +16,4 @@ psycopg2-binary
 redis
 pyjwt
 sentry-sdk
+starlette_exporter

requirements.pip

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile --output-file=requirements.pip requirements.in
@@ -49,10 +49,10 @@ dnspython==2.0.0
     # via email-validator
 email-validator==1.1.1
     # via fastapi
-fastapi-cache==0.0.5
-    # via -r requirements.in
 fastapi[all]==0.61.1
     # via -r requirements.in
+fastapi-cache==0.0.5
+    # via -r requirements.in
 formencode==2.0.0
     # via pyxform
 graphene==2.1.8
@@ -104,6 +104,8 @@ orjson==3.4.2
     # via fastapi
 pandas==1.1.4
     # via -r requirements.in
+prometheus-client==0.11.0
+    # via starlette-exporter
 promise==2.3
     # via
     #   graphql-core
@@ -143,12 +145,12 @@ requests==2.24.0
     #   tableauserverclient
 rfc3986[idna2008]==1.4.0
     # via httpx
-rq-scheduler==0.10.0
-    # via -r requirements.in
 rq==1.6.1
     # via
     #   -r requirements.in
     #   rq-scheduler
+rq-scheduler==0.10.0
+    # via -r requirements.in
 rx==1.6.1
     # via graphql-core
 s3transfer==0.3.3
@@ -174,7 +176,11 @@ sqlalchemy==1.3.20
     #   -r requirements.in
     #   alembic
 starlette==0.13.6
-    # via fastapi
+    # via
+    #   fastapi
+    #   starlette-exporter
+starlette-exporter==0.9.0
+    # via -r requirements.in
 tableauhyperapi==0.0.11556
     # via -r requirements.in
 tableauserverclient==0.13