核心模块

Spark Core

Spark SQL

Spark Streaming

Spark MLlib

Spark GraphX

 快速上手

创建Maven项目，添加Scala插件

【Idea】建立Scale框架_岱宗夫如何、的博客-CSDN博客

POM.xml,添加Spark依赖环境和 scala插件

scala-maven-plugin 用于在 maven 中编译/测试/运行/记录 scala 代码。

<dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.0.0</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <!-- 该插件用于将 Scala 代码编译成 class 文件 -->
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.2</version>
                <executions>
                    <execution>
                        <!-- 声明绑定到 maven 的 compile 阶段 -->
                        <goals>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.1.0</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

WordCount案例

package com.qihang.bigdata.spark.core.wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object WordCount {
  def main(args: Array[String]): Unit = {
    //Application
    //Spark 框架
    //TODO 建立和Spark框架连接
    val sparConf = new SparkConf().setMaster("local").setAppName("WordCount")
    val sc = new SparkContext(sparConf)
    //TODO 执行业务逻辑
    //读取文件，获取一行一行的数据
    //Hello world
    //Hello Spark
    val lines = sc.textFile("datas")
    //拆分成一个一个的单词 扁平化
    //hello world hello,world,hello,world
    val words = lines.flatMap(_.split(" "))
    //按照word分组
    //(hello, hello, hello)
    val wordGroup: RDD[(String, Iterable[String])] = words.groupBy(word => word)
    //转换
    //(hello,3)
//    val wordToCount: RDD[(String, Int)] = wordGroup.map(
//      tuple => (tuple._1, tuple._2.size)
//    )
    val wordToCount: RDD[(String, Int)] = wordGroup.map{
      case ( word, list ) => {
        (word, list.size)
      }
    }
    //打印
    val array = wordToCount.collect()
    array.foreach(println(_))
    //TODO 关闭连接
    sc.stop()
  }

}

log4j.rootCategory=ERROR, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd 
HH:mm:ss} %p %c{1}: %m%n
# Set the default spark-shell log level to ERROR. When running the spark-shell, 
the
# log level for this class is used to overwrite the root logger's log level, so 
that
# the user can have different defaults for the shell and regular Spark apps.
log4j.logger.org.apache.spark.repl.Main=ERROR
# Settings to quiet third party logs that are too verbose
log4j.logger.org.spark_project.jetty=ERROR
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR
log4j.logger.org.apache.parquet=ERROR
log4j.logger.parquet=ERROR
# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent 
UDFs in SparkSQL with Hive support
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR