Updated makefile and README

Signed-off-by: Yash Kalathiya <[email protected]>

.make.versions

@@ -74,7 +74,8 @@ INGEST_TO_PARQUET_VERSION=$(DPK_VERSION)
 KFP_DOCKER_VERSION=$(DPK_VERSION)
 KFP_DOCKER_VERSION_v2=$(DPK_VERSION)
 
-LEGAL_REMOVAL_VERSION=$(DPK_VERSION) 
+LEGAL_REMOVAL_PYTHON_VERSION=$(DPK_VERSION) 
+LEGAL_REMOVAL_RAY_VERSION=$(DPK_VERSION) 
 
 ################## ################## ################## ################## ################## ##################
 # Begin versions that the repo depends on. 

transforms/code/legal_removal/Makefile

@@ -43,4 +43,16 @@ test-src::
 
 load-image::
 	@# Help: Recursively make $@ in all subdirs 
-	$(MAKE) RULE=$@ .recurse
+	$(MAKE) RULE=$@ .recurse
+
+.PHONY: workflow-venv
+workflow-venv:
+	$(MAKE) -C kfp_ray workflow-venv
+
+.PHONY: workflow-test
+workflow-test:
+	$(MAKE) -C kfp_ray workflow-test
+
+.PHONY: workflow-upload
+workflow-upload:
+	$(MAKE) -C kfp_ray workflow-upload

transforms/code/legal_removal/python/Makefile

@@ -8,7 +8,7 @@ REPOROOT=../../../..
 # $(REPOROOT)/.make.versions file contains the versions
 
 TRANSFORM_NAME=legal_removal
-DOCKER_IMAGE_VERSION=${LEGAL_REMOVAL_VERSION}
+DOCKER_IMAGE_VERSION=${LEGAL_REMOVAL_PYTHON_VERSION}
 
 include $(REPOROOT)/transforms/.make.transforms
 
@@ -34,7 +34,7 @@ setup:: .transforms.setup
 
 # distribution versions is the same as image version.
 set-versions:
-	$(MAKE) TRANSFORM_PYTHON_VERSION=${FILTER_PYTHON_VERSION} TOML_VERSION=$(DOCKER_IMAGE_VERSION) .transforms.set-versions 
+	$(MAKE) TRANSFORM_PYTHON_VERSION=${LEGAL_REMOVAL_PYTHON_VERSION} TOML_VERSION=$(DOCKER_IMAGE_VERSION) .transforms.set-versions 
 
 build-dist:: set-versions .defaults.build-dist 
 

transforms/code/legal_removal/python/README.md

@@ -15,8 +15,8 @@ After detecting license and copyright position code has been stored at same colu
 The set of dictionary keys holding configuration for values are as follows:
 
 * --legal_removal_contents_column_name - specifies the column name which holds code content. By default the value is 'contents'.
-* --legal_removal_license - specifies the bool value for removing license or not. Default value is True.
-* --legal_removal_copyright - specifies the bool value for removing copyright or not. Default value is True. 
+* --legal_removal_license - specifies the bool value for removing license or not. Write true or false.  Default value is True.
+* --legal_removal_copyright - specifies the bool value for removing copyright or not. Write true or false. Default value is True. 
 
 ## Running
 You can run the [legal_removal_local.py](src/legal_removal_local.py) (python-only implementation) or [legal_removal_local_ray.py](ray/src/legal_removal_local_ray.py) (ray-based  implementation) to transform the `test1.parquet` file in [test input data](test-data/input) to an `output` directory.  The directory will contain both the new annotated `test1.parquet` file and the `metadata.json` file.
@@ -26,8 +26,7 @@ You can run the [legal_removal_local.py](src/legal_removal_local.py) (python-onl
 ### Launched Command Line Options 
 When running the transform with the Ray launcher (i.e. TransformLauncher),
 the following command line arguments are available in addition to 
-the options provided by the [ray launcher](../../../../data-processing-lib/doc/ray-launcher-options.md)
-and the [python launcher](../../../../data-processing-lib/doc/python-launcher-options.md).
+the [python launcher](../../../../data-processing-lib/doc/python-launcher-options.md).
 
 ### Running the samples
 To run the samples, use the following `make` targets

transforms/code/legal_removal/python/src/legal_removal_transform.py

@@ -19,7 +19,7 @@
     PythonTransformRuntimeConfiguration,
 )
 from data_processing.transform import AbstractTableTransform, TransformConfiguration
-from data_processing.utils import TransformUtils,get_logger,CLIArgumentProvider
+from data_processing.utils import get_logger,CLIArgumentProvider
 from scancode import api
 
 logger = get_logger(__name__)

transforms/code/legal_removal/ray/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/rayproject/ray:2.9.3-py310
+FROM docker.io/rayproject/ray:2.24.0-py310
 
 # install pytest
 RUN pip install --no-cache-dir pytest

transforms/code/legal_removal/ray/Makefile

@@ -6,7 +6,7 @@ include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=legal_removal
 # $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${LEGAL_REMOVAL_VERSION}
+DOCKER_IMAGE_VERSION=${LEGAL_REMOVAL_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 
@@ -28,13 +28,8 @@ publish:: publish-dist publish-image
 
 publish-image:: .transforms.publish-image-ray
 
-# distribution versions is the same as image version.
-set-versions:
-	$(MAKE) TOML_VERSION=$(DOCKER_IMAGE_VERSION) .defaults.update-toml
-	cat pyproject.toml | sed                                                                        \
-                -e 's/"dpk-license_copyright_removal-transform-python\(..\).*",/"dpk-license_copyright_removal-transform-python\1$(LICENSE_COPYRIGHT_REMOVAL_VERSION)",/'   \
-                > tt.toml
-	mv tt.toml pyproject.toml
+set-versions: 
+	$(MAKE) TRANSFORM_PYTHON_VERSION=${LEGAL_REMOVAL_PYTHON_VERSION} TOML_VERSION=$(LEGAL_REMOVAL_RAY_VERSION) .transforms.set-versions 
 
 build-dist:: set-versions .defaults.build-dist 
 

transforms/code/legal_removal/ray/README.md

@@ -12,11 +12,7 @@ After detecting license and copyright position code has been stored at same colu
 
 ## Configuration and command line Options
 
-The set of dictionary keys holding configuration for values are as follows:
-
-* --legal_removal_contents_column_name - specifies the column name which holds code content. By default the value is 'contents'.
-* --legal_removal_license - specifies the bool value for removing license or not. Default value is True.
-* --legal_removal_copyright - specifies the bool value for removing copyright or not. Default value is True. 
+This project wraps the [license removal transform](../python) with a Ray runtime.
 
 ## Running