Merge pull request #24 from onaio/22-export-configuration-changes

Allow export settings to be configured via the API

.github/workflows/ci.yml

@@ -25,6 +25,19 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_PASSWORD: duva
+          POSTGRES_DB: duva
+          POSTGRES_USER: duva
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v2
       - name: Setup python

README.md

@@ -44,6 +44,13 @@ $ pip install -r requirements.pip
 $ pip install -r dev-requirements.pip
 ```
 
+5. Create postgres user & database for the application
+
+```sh
+$ psql -c "CREATE USER duva WITH PASSWORD 'duva';"
+$ psql -c "CREATE DATABASE duva OWNER duva;"
+```
+
 At this point the application can be started. _Note: Ensure the redis server has been started_
 
 ```

app/alembic/versions/259f90f13bd6_add_export_settings_column.py

@@ -0,0 +1,36 @@
+"""Add export_settings column
+
+Revision ID: 259f90f13bd6
+Revises: 2b468eeb193d
+Create Date: 2021-10-04 12:29:35.298536
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "259f90f13bd6"
+down_revision = "2b468eeb193d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "configuration",
+        sa.Column(
+            "export_settings",
+            sa.JSON(),
+            server_default='{"include_labels": true, "remove_group_name": true, "do_not_split_select_multiple": false, "include_reviews": false, "include_labels_only": true, "value_select_multiples": true}',
+            nullable=False,
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("configuration", "export_settings")
+    # ### end Alembic commands ###

app/models.py

@@ -1,4 +1,5 @@
 import sqlalchemy.types as types
+import json
 from typing import Optional
 from cryptography.fernet import Fernet
 from sqlalchemy import (
@@ -219,6 +220,20 @@ class Configuration(ModelMixin, EncryptionMixin, Base):
     token_value = Column(String)
     project_name = Column(String, default="default")
     user = Column(Integer, ForeignKey("user.id", ondelete="CASCADE"))
+    export_settings = Column(
+        JSON,
+        nullable=False,
+        server_default=json.dumps(
+            {
+                "include_labels": True,
+                "remove_group_name": True,
+                "do_not_split_select_multiple": False,
+                "include_reviews": False,
+                "include_labels_only": True,
+                "value_select_multiples": True,
+            }
+        ),
+    )
 
     @classmethod
     def filter_using_user_id(cls, db: Session, user_id: int):

app/routers/configuration.py

@@ -77,14 +77,7 @@ def create_configuration(
     Create a new Tableau Server Configuration that can be attached
     to a hyper file to define where the hyper file should be pushed to.
     """
-    config_data = schemas.ConfigurationCreate(
-        user=user.id,
-        server_address=config_data.server_address,
-        site_name=config_data.site_name,
-        token_name=config_data.token_name,
-        token_value=config_data.token_value,
-        project_name=config_data.project_name,
-    )
+    config_data = schemas.ConfigurationCreate(user=user.id, **config_data.dict())
     try:
         config = Configuration.create(db, config_data)
         return config

app/schemas.py

@@ -65,12 +65,22 @@ class FileCreate(FileBase):
     meta_data: dict = {SYNC_FAILURES_METADATA: 0, JOB_ID_METADATA: ""}
 
 
+class ExportConfigurationSettings(BaseModel):
+    include_labels: Optional[bool] = True
+    remove_group_name: Optional[bool] = True
+    do_not_split_select_multiple: Optional[bool] = False
+    include_reviews: Optional[bool] = False
+    include_labels_only: Optional[bool] = True
+    value_select_multiples: Optional[bool] = True
+
+
 class ConfigurationResponse(BaseModel):
     id: int
     server_address: str
     site_name: str
     token_name: str
     project_name: str
+    export_settings: ExportConfigurationSettings
 
     class Config:
         orm_mode = True
@@ -82,6 +92,7 @@ class ConfigurationListResponse(BaseModel):
     site_name: str
     token_name: str
     project_name: str
+    export_settings: ExportConfigurationSettings
 
     class Config:
         orm_mode = True
@@ -93,13 +104,18 @@ class ConfigurationCreateRequest(BaseModel):
     token_name: str
     project_name: str
     token_value: str
+    export_settings: Optional[
+        ExportConfigurationSettings
+    ] = ExportConfigurationSettings()
 
 
 class ConfigurationPatchRequest(BaseModel):
     server_address: Optional[str]
     site_name: Optional[str]
     token_name: Optional[str]
     project_name: Optional[str]
+    token_value: Optional[str]
+    export_settings: Optional[ExportConfigurationSettings]
 
 
 class ConfigurationCreate(ConfigurationCreateRequest):

app/settings.py

@@ -7,7 +7,7 @@ class Settings(BaseSettings):
     app_version: str = "v0.0.5"
     app_host: str = "127.0.0.1"
     app_port: int = 8000
-    database_url: str = "sqlite:///./sqllite_db.db"
+    database_url: str = "postgres://duva:[email protected]/duva"
     debug: bool = True
     sentry_dsn: str = ""
     session_same_site: str = "none"

app/tests/routes/test_configuration.py

@@ -45,6 +45,7 @@ def test_create_retrieve_config(self, create_user_and_login):
             token_name="test",
             project_name="default",
             id=config_id,
+            export_settings=schemas.ExportConfigurationSettings(),
         ).dict()
         assert response.json() == expected_data
 
@@ -88,6 +89,7 @@ def test_patch_config(self, create_user_and_login):
             token_name="test",
             project_name="default",
             id=config_id,
+            export_settings=schemas.ExportConfigurationSettings(),
         ).dict()
 
         # Able to patch Tableau Configuration

app/tests/utils/test_onadata_utils.py

@@ -8,7 +8,7 @@
 from httpx._models import Response
 
 from app import schemas
-from app.models import Server, User, HyperFile
+from app.models import Server, User, HyperFile, Configuration
 from app.tests.test_base import TestBase
 from app.utils.onadata_utils import (
     get_access_token,
@@ -67,12 +67,25 @@ def test_get_csv_export(
         """
         user, _ = create_user_and_login
         server = user.server
+        configuration = Configuration.create(
+            self.db,
+            schemas.ConfigurationCreate(
+                user=user.id,
+                server_address="http://testserver",
+                site_name="test",
+                token_name="test",
+                token_value="test",
+                project_name="test",
+            ),
+        )
         hyperfile = HyperFile.create(
             self.db,
             schemas.FileCreate(
                 user=user.id, filename="test.hyper", is_active=True, form_id="111"
             ),
         )
+        hyperfile.configuration_id = configuration.id
+        self.db.commit()
         file_mock = MagicMock()
         file_mock.name = "/tmp/test"
         mock_httpx_client().__enter__().get.return_value = Response(
@@ -87,12 +100,17 @@ def test_get_csv_export(
                 user,
                 user.server,
                 self.db,
-                export_configuration={"include_labels": "true"},
             )
             assert ret == Path(file_mock.name)
             mock_get_access_token.assert_called_with(user, server, self.db)
+            expected_query_params = "&".join(
+                [
+                    f"{param}={value}"
+                    for param, value in configuration.export_settings.items()
+                ]
+            )
             mock_get_csv_export.assert_called_with(
-                f"{server.url}/api/v1/forms/{hyperfile.form_id}/export_async.json?format=csv&include_labels=true",
+                f"{server.url}/api/v1/forms/{hyperfile.form_id}/export_async.json?format=csv&{expected_query_params}",
                 mock_httpx_client().__enter__(),
             )
 

app/utils/onadata_utils.py

@@ -144,7 +144,6 @@ def get_csv_export(
     user: schemas.User,
     server: schemas.Server,
     db: SessionLocal,
-    export_configuration: dict = None,
 ) -> str:
     """
     Retrieves a CSV Export for an XForm linked to a Hyperfile
@@ -159,9 +158,9 @@ def get_csv_export(
         resp = client.get(form_url + ".json")
         if resp.status_code == 200:
             url = f"{form_url}/export_async.json?format=csv"
-
-            if export_configuration:
-                for key, value in export_configuration.items():
+            if hyperfile.configuration:
+                export_settings = hyperfile.configuration.export_settings
+                for key, value in export_settings.items():
                     url += f"&{key}={value}"
 
             csv_export = _get_csv_export(url, client)
@@ -174,12 +173,6 @@ def start_csv_import_to_hyper(
     hyperfile_id: int,
     process: HyperProcess,
     schedule_cron: bool = True,
-    include_labels: bool = True,
-    remove_group_name: bool = True,
-    do_not_split_select_multiples: bool = True,
-    include_reviews: bool = False,
-    include_images: bool = False,
-    include_labels_only: bool = True,
 ):
     """
     Starts a CSV Export importation process that imports CSV Data into
@@ -189,13 +182,6 @@ def start_csv_import_to_hyper(
     hyperfile_id :: int : A unique identifier for a HyperFile object
     schedule_cron :: bool : Whether to schedule a cron job that triggers a CSV Import
                             periodically.
-    include_labels :: bool : Whether to request an OnaData CSV Export with labels included as headers
-                             and values
-    remove_group_names :: bool : Whether to request an OnaData CSV Export without the column group names included in the header
-    do_not_split_select_multiples :: bool : Whether to request an OnaData CSV Export that doesn't splict select multiples into different columns
-    include_reviews :: bool : Whether to request an OnaData CSV Export that includes review
-    include_images :: bool : Whether to request an OnaData CSV Export that includes image URLs
-    include_labels_only :: bool : Whether to request an OnaData CSV Export that includes labels only
     """
     db = SessionLocal()
     redis_client = Redis(
@@ -216,22 +202,11 @@ def start_csv_import_to_hyper(
                 db.refresh(hyperfile)
 
                 try:
-                    export_configuration = {
-                        "include_labels": str(include_labels).lower(),
-                        "remove_group_name": str(remove_group_name).lower(),
-                        "do_not_split_select_multiples": str(
-                            do_not_split_select_multiples
-                        ).lower(),
-                        "include_reviews": str(include_reviews).lower(),
-                        "include_images": str(include_images).lower(),
-                        "include_labels_only": str(include_labels_only).lower(),
-                    }
                     export = get_csv_export(
                         hyperfile,
                         user,
                         server,
                         db,
-                        export_configuration=export_configuration,
                     )
 
                     if export:

docker-compose.yml

@@ -37,7 +37,7 @@ services:
       - CRON_SCHEDULE=*/30 * * * *
       - DEBUG=True
       - DATABASE_URL=postgresql://duva:duva@db/duva
-      - RUN_MIGRATION=True
+      - RUN_MIGRATIONS=True
   scheduler:
     build:
       context: .