From 81baf106dba167a81908b29ef12fdac140101f13 Mon Sep 17 00:00:00 2001
From: SelimHorri <horri.slim@gmail.com>
Date: Thu, 21 Jan 2021 01:21:49 +0100
Subject: [PATCH] create a dataframe, manipulate it and load it in a mysql
 database locally

---
 .classpath                                  | 26 +++---
 .gitignore                                  | 33 +++++++
 pom.xml                                     | 98 +++++++++++----------
 src/main/java/com/selimhorri/pack/Main.java | 22 +++++
 src/main/resources/name_job.txt             |  5 ++
 5 files changed, 123 insertions(+), 61 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 src/main/resources/name_job.txt

diff --git a/.classpath b/.classpath
index 6e0021f..595d245 100644
--- a/.classpath
+++ b/.classpath
@@ -1,23 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
-	<classpathentry kind="src" output="target/classes" path="src/main/java">
-		<attributes>
-			<attribute name="optional" value="true"/>
-			<attribute name="maven.pomderived" value="true"/>
-		</attributes>
-	</classpathentry>
 	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
-		<attributes>
-			<attribute name="optional" value="true"/>
-			<attribute name="maven.pomderived" value="true"/>
-			<attribute name="test" value="true"/>
-		</attributes>
-	</classpathentry>
 	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
@@ -42,6 +29,19 @@
 			<attribute name="m2e-apt" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry kind="src" output="target/classes" path="src/main/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
 		<attributes>
 			<attribute name="optional" value="true"/>
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..549e00a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,33 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
diff --git a/pom.xml b/pom.xml
index f787e2d..3a1556b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,58 +13,62 @@
 		<maven.compiler.source>11</maven.compiler.source>
 		<maven.compiler.target>11</maven.compiler.target>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<scala.version>2.12</scala.version>
+	      <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+		<scala.version>2.11</scala.version>
 		<spark.version>2.3.1</spark.version>
 	</properties>
 	
- <dependencies>
-    <!-- Spark -->
-	<dependency>
-		<groupId>org.apache.spark</groupId>
-		<artifactId>spark-core_${scala.version}</artifactId>
-		<version>${spark.version}</version>
-	</dependency>
-	<dependency>
-		<groupId>org.apache.spark</groupId>
-		<artifactId>spark-sql_${scala.version}</artifactId>
-		<version>${spark.version}</version>
-		<exclusions>
-			<exclusion>
-				<groupId>org.slf4j</groupId>
-				<artifactId>slf4j-simple</artifactId>
-			</exclusion>
-		</exclusions>
-	</dependency>
-	<dependency>
-		<groupId>org.apache.spark</groupId>
-		<artifactId>spark-mllib_${scala.version}</artifactId>
-		<version>${spark.version}</version>
-		<exclusions>
-			<exclusion>
-				<groupId>org.slf4j</groupId>
-				<artifactId>slf4j-log4j12</artifactId>
-			</exclusion>
-			<exclusion>
-				<groupId>org.slf4j</groupId>
-				<artifactId>slf4j-simple</artifactId>
-			</exclusion>
-		</exclusions>
-	</dependency>
-
-	<dependency>
-		<groupId>junit</groupId>
-		<artifactId>junit</artifactId>
-		<version>4.11</version>
-		<scope>test</scope>
-	</dependency>
-    
+	<dependencies>
+	  <!-- Spark -->
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-core_${scala.version}</artifactId>
+			<version>${spark.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-sql_${scala.version}</artifactId>
+			<version>${spark.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-simple</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.spark</groupId>
+			<artifactId>spark-mllib_${scala.version}</artifactId>
+			<version>${spark.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-log4j12</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-simple</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+		    <groupId>mysql</groupId>
+		    <artifactId>mysql-connector-java</artifactId>
+		    <version>8.0.22</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.11</version>
+			<scope>test</scope>
+		</dependency>
+	  
 	</dependencies>
 
 	<build>
 	
 		<plugins>
-			<plugin>
+			<!-- <plugin>
 			    <groupId>org.apache.maven.plugins</groupId>
 			    <artifactId>maven-dependency-plugin</artifactId>
 			    <executions>
@@ -81,7 +85,7 @@
 			            </configuration>
 			        </execution>
 			    </executions>
-			</plugin>
+			</plugin> -->
 			
 			<plugin>
 			    <groupId>org.springframework.boot</groupId>
@@ -92,15 +96,13 @@
 			                <goal>repackage</goal>
 			            </goals>
 			            <configuration>
-			
 			                <mainClass>com.selimhorri.pack.Main</mainClass>
-			
 			            </configuration>
 			        </execution>
 			    </executions>
 			</plugin>
-		
 		</plugins>
+		<finalName>spark-app</finalName>
 	</build>
 
 </project>
diff --git a/src/main/java/com/selimhorri/pack/Main.java b/src/main/java/com/selimhorri/pack/Main.java
index 370329a..713045a 100644
--- a/src/main/java/com/selimhorri/pack/Main.java
+++ b/src/main/java/com/selimhorri/pack/Main.java
@@ -1,10 +1,32 @@
 package com.selimhorri.pack;
 
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+import static org.apache.spark.sql.functions.*;
+
+import java.util.Properties;
+
 public class Main {
 	
 	public static void main(String[] args) {
 		
+		final SparkSession sparkSession = new SparkSession.Builder().appName("CSV to DB").master("local").getOrCreate();
+		
+		Dataset<Row> df = sparkSession.read().format("csv").option("header", true).load("src/main/resources/name_job.txt");
+		
+		df = df.withColumn("fullName", concat(lit("FIRST_NAME => "), df.col("first_name"), lit(" || "), lit("LAST_NAME => "), df.col("last_name")) );
+		df.show();
+		
+		final String dbUrl = "jdbc:mysql://localhost:3306/spark_db";
+		final Properties properties = new Properties();
+		// properties.setProperty("driver", "");
+		properties.setProperty("user", "root");
+		properties.setProperty("password", "");
 		
+		df.write().mode(SaveMode.Overwrite).jdbc(dbUrl, "persons", properties);
+		System.out.println("====>> Loaded in the Database <<====");
 		
 	}
 	
diff --git a/src/main/resources/name_job.txt b/src/main/resources/name_job.txt
new file mode 100644
index 0000000..fcff170
--- /dev/null
+++ b/src/main/resources/name_job.txt
@@ -0,0 +1,5 @@
+first_name,last_name,job
+Selim,Horri,Software Engineer
+Amine,Ajimi,Student
+Omar,Derouiche,Agent
+Yesmine,Derouiche,Student
\ No newline at end of file