id-links-csv.sh

# Submit a spark job against a Mesos-Spark configuration.
# see http://spark.apache.org/docs/latest/running-on-mesos.html
#
# SPARK_DATA: where datasets are 
# SPARK_JOBS: where spark job jars sit
# SPARK_HOME: where apache has been installed
#

DATASET_DIR=file:///home/int/data/gbif-100k

$SPARK_HOME/bin/spark-submit \
--master mesos://apihack-c18.idigbio.org:7077 \
--deploy-mode cluster \
--executor-memory 6G \
--driver-memory 6G \
--total-executor-cores 5 \
--class LinkIdentifiersJob \
file:///home/int/jobs/iDigBio-LD-assembly-1.3.1.jar \
-o $DATASET_DIR/id_links.txt \
-f com.databricks.spark.csv \
$DATASET_DIR/occurrence.txt.parquet