<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<parent>
		<groupId>io.github.solven-eu.pepper</groupId>
		<artifactId>pepper-hadoop-parent</artifactId>
		<version>4.2</version>
	</parent>

	<artifactId>pepper-spark-parent</artifactId>
	<packaging>pom</packaging>

	<modules>
		<module>spark-library</module>
		<module>spark_2.10</module>
		<module>spark_2.11</module>
		<module>spark_2.12</module>
		<module>spark_2.13</module>

		<!-- Helps building custom Spark docker image -->
		<module>spark-docker</module>

		<!-- Demonstrate how to package a Spark job with SpringBoot -->
		<module>spark-springboot</module>
	</modules>

	<properties>
		<scala.shortVersion>2.11</scala.shortVersion>
		<scala.version>2.11</scala.version>
		<!-- https://mvnrepository.com/artifact/org.scala-lang/scala-reflect -->
		<scala-lang.version>2.11.12</scala-lang.version>

		<!-- Most Spark distribution has this quite old Guava: as it easily leads -->
		<!-- to conflicts, we make sure we compile locally with this library -->
		<guava.version>14.0.1</guava.version>

		<spark.shortVersion>2.4</spark.shortVersion>
		<spark.version>2.4.8</spark.version>

		<parquet.version>1.12.3</parquet.version>

		<!-- 3.1.2 leads to org.codehaus.janino.InternalCompilerException -->
		<!-- <version>3.1.2</version> -->
		<!-- <version>3.1.6</version> -->
		<!-- Indeed, moved to org.codehaus.commons.compiler.InternalCompilerException -->
		<janino.version>3.0.16</janino.version>
	</properties>

	<dependencyManagement>
		<dependencies>
			<dependency>
				<!-- https://stackoverflow.com/questions/42352091/spark-sql-fails-with-java-lang-noclassdeffounderror-org-codehaus-commons-compil -->
				<groupId>org.codehaus.janino</groupId>
				<artifactId>commons-compiler</artifactId>
				<version>${janino.version}</version>
			</dependency>
			<dependency>
				<!-- https://stackoverflow.com/questions/42352091/spark-sql-fails-with-java-lang-noclassdeffounderror-org-codehaus-commons-compil -->
				<groupId>org.codehaus.janino</groupId>
				<artifactId>janino</artifactId>
				<version>${janino.version}</version>
			</dependency>

			<dependency>
				<groupId>org.apache.spark</groupId>
				<artifactId>spark-core_${scala.version}</artifactId>
				<version>${spark.version}</version>
			</dependency>
			<dependency>
				<groupId>org.apache.spark</groupId>
				<artifactId>spark-hive_${scala.version}</artifactId>
				<version>${spark.version}</version>
				<exclusions>
					<exclusion>
						<!-- Hive-exec is necessary to instantiate a SparkSession with HiveSupport -->
						<!-- However, we may need a recent Hive-exec to enable recent Hadoop, to enable recent ABFSS features -->
						<!-- https://stackoverflow.com/questions/39444493/how-to-create-sparksession-with-hive-support-fails-with-hive-classes-are-not-f -->
						<groupId>org.spark-project.hive</groupId>
						<artifactId>hive-exec</artifactId>
					</exclusion>
				</exclusions>
			</dependency>
			<dependency>
				<!-- We add hive-common manually as we rejected the hive-exec@Spark to enabler a more recent version -->
				<groupId>org.apache.hive</groupId>
				<artifactId>hive-common</artifactId>
				<version>3.1.2</version>
			</dependency>

			<dependency>
				<groupId>org.scala-lang</groupId>
				<artifactId>scala-reflect</artifactId>
				<version>${scala-lang.version}</version>
			</dependency>
			<dependency>
				<groupId>org.scala-lang</groupId>
				<artifactId>scala-library</artifactId>
				<version>${scala-lang.version}</version>
			</dependency>
			<dependency>
				<groupId>org.scala-lang</groupId>
				<artifactId>scalap</artifactId>
				<version>${scala-lang.version}</version>
			</dependency>
			<!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
			<dependency>
				<groupId>org.apache.parquet</groupId>
				<artifactId>parquet-hadoop</artifactId>
				<version>${parquet.version}</version>
			</dependency>
			<!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-column -->
			<dependency>
				<groupId>org.apache.parquet</groupId>
				<artifactId>parquet-column</artifactId>
				<version>${parquet.version}</version>
			</dependency>
			<dependency>
				<groupId>org.apache.parquet</groupId>
				<artifactId>parquet-common</artifactId>
				<version>${parquet.version}</version>
			</dependency>
			<dependency>
				<groupId>org.apache.parquet</groupId>
				<artifactId>parquet-avro</artifactId>
				<version>${parquet.version}</version>
			</dependency>

		</dependencies>
	</dependencyManagement>

	<profiles>
		<!-- https://stackoverflow.com/questions/43883325/scala-spark-version-compatibility -->
		<profile>
			<id>scala-2.11</id>
			<properties>
				<scala.shortVersion>2.11</scala.shortVersion>
				<scala.version>2.11</scala.version>
				<!-- https://mvnrepository.com/artifact/org.scala-lang/scala-reflect -->
				<scala-lang.version>2.11.12</scala-lang.version>
			</properties>
		</profile>
		<profile>
			<id>scala-2.12</id>
			<properties>
				<scala.shortVersion>2.12</scala.shortVersion>
				<scala.version>2.12</scala.version>
				<!-- https://mvnrepository.com/artifact/org.scala-lang/scala-reflect -->
				<scala-lang.version>2.12.14</scala-lang.version>
			</properties>
		</profile>

		<!-- https://stackoverflow.com/questions/28458058/maven-shade-plugin-exclude-a-dependency-and-all-its-transitive-dependencies -->
		<!-- In these profile, we switch Spark dependencies as provided, so they are not included in the shadedJar (without having to list transitive dependencies 
			individually) -->
		<profile>
			<id>spark-2.4</id>
			<properties>
				<spark.shortVersion>2.4</spark.shortVersion>
				<spark.version>2.4.8</spark.version>
			</properties>
			<dependencies>
				<!-- In this profile, we switch Spark dependencies as provided, so they are not included in the shadedJar (without having to list transitive dependencies 
					individually) -->
				<dependency>
					<groupId>org.apache.spark</groupId>
					<artifactId>spark-core_${scala.version}</artifactId>
					<version>${spark.version}</version>
					<scope>provided</scope>
				</dependency>
				<dependency>
					<groupId>org.apache.spark</groupId>
					<artifactId>spark-sql_${scala.version}</artifactId>
					<version>${spark.version}</version>
					<scope>provided</scope>
				</dependency>
			</dependencies>
		</profile>
		<profile>
			<!-- https://stackoverflow.com/questions/43883325/scala-spark-version-compatibility -->
			<id>spark-3.1</id>
			<properties>
				<!-- Spark3.1 requires Scala2.12+ -->
				<!-- Pick Scala version through -Pscala-2.1X -->
				<spark.shortVersion>3.1</spark.shortVersion>
				<spark.version>3.1.2</spark.version>
			</properties>
			<dependencies>
				<!-- In this profile, we switch Spark dependencies as provided, so they are not included in the shadedJar (without having to list transitive dependencies 
					individually) -->
				<dependency>
					<groupId>org.apache.spark</groupId>
					<artifactId>spark-core_${scala.version}</artifactId>
					<version>${spark.version}</version>
					<scope>provided</scope>
				</dependency>
				<dependency>
					<groupId>org.apache.spark</groupId>
					<artifactId>spark-sql_${scala.version}</artifactId>
					<version>${spark.version}</version>
					<scope>provided</scope>
				</dependency>
			</dependencies>
		</profile>
	</profiles>
</project>