基于hive分析Flask为后端框架echarts为前端框架的招聘网站可视化大屏项目
1. 项目概述
项目目标是构建一个大数据分析系统,包含以下核心模块:
1、数据爬取:通过request请求获取猎聘网的就业数据。
2、数据存储和分析:使用 Hive 进行数据存储和分析。
3、数据迁移:使用sqoop将hive数据导入mysql。
4、后端服务:使用 Flask 搭建数据接口,将分析结果提供给前端。
5、数据可视化:使用 ECharts 制作大屏展示,实现数据的图形化呈现。
2. 项目环境准备
在开始之前,需要搭建如下环境:
Hive:作为数据仓库,用于存储和分析数据。
Flask:轻量级 Python Web 框架,用于构建后端 RESTful API。
ECharts:JavaScript 图表库,用于前端数据可视化。
MySQL:用于保存一些系统配置或小规模数据。
Sqoop:数据同步工具,将hive数据同步到mysql。
3、数据爬取
通过python获取猎聘网的照片信息,存储到csv文件里
import csv
import time
import requests
import execjs
from storage.csv2mysql import sync_data2db
f = open('../storage/data.csv', mode='a', encoding='utf-8')
csv_writer = csv.DictWriter(f,fieldnames=[
'职位',
'城市',
'薪资',
'经验',
'标签',
'公司',
'公司领域',
'公司规模'])
csv_writer.writeheader()
def read_js_code():
f= open('/Users/shareit/workspace/chart_show/demo.js',encoding='utf-8')
txt = f.read()
js_code = execjs.compile(txt)
ckId = js_code.call('r',32)
return ckId
def post_data():
read_js_code()
url = "https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job"
headers = {
'Accept': 'application/json, text/plain, */*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Sec-Ch-Ua-Platform':'macOS',
'Content-Length': '398',
'Content-Type': 'application/json;charset=UTF-8;',
'Host': 'api-c.liepin.com',
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
'Origin': 'https://www.liepin.com',
'Referer': 'https://www.liepin.com/',
'Sec-Ch-Ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-site',
'X-Client-Type': 'web',
'X-Fscp-Bi-Stat': '{"location": "https://www.liepin.com/zhaopin"}',
'X-Fscp-Fe-Version': '',
'X-Fscp-Std-Info': '{"client_id": "40108"}',
'X-Fscp-Trace-Id': '52262313-e6ca-4cfd-bb67-41b4a32b8bb5',
'X-Fscp-Version': '1.1',
'X-Requested-With': 'XMLHttpRequest',
}
list = ["H01$H0001","H01$H0002",
"H01$H0003","H01$H0004","H01$H0005",
"H01$H0006","H01$H0007","H01$H0008",
"H01$H0009","H01$H00010","H02$H0018","H02$H0019","H03$H0022",
"H03$H0023","H03$H0024","H03$H0025","H04$H0030","H04$H0031",
"H04$H0032","H05$H05","H06$H06","H07$H07","H08$H08"]
for name in list:
print("-------{}---------".format(name))
for i in range(10):
print("------------第{}页-----------".format(i))
data = {"data": {"mainSearchPcConditionForm":
{"city": "410", "dq": "410", "pubTime": "", "currentPage": i, "pageSize": 40, "key": "",
"suggestTag": "", "workYearCode": "1", "compId": "", "compName": "", "compTag": "",
"industry": name, "salary": "", "jobKind": "", "compScale": "", "compKind": "", "compStage": "",
"eduLevel": ""},
"passThroughForm":
{"scene": "page", "skId": "z33lm3jhwza7k1xjvcyn8lb8e9ghxx1b",
"fkId": "z33lm3jhwza7k1xjvcyn8lb8e9ghxx1b",
"ckId": read_js_code(),
'sfrom': 'search_job_pc'}}}
response = requests.post(url=url, json=data, headers=headers)
time.sleep(2)
parse_data(response)
def parse_data(response):
try:
jobCardList = response.json()['data']['data']['jobCardList']
except Exception as e:
return
4、加载hive数据进行分析
1、将storage下的data.csv上传到虚拟机上
2、创建work_base表,并将data.csv数据加载到hive表里
CREATE TABLE work_base (
id INT COMMENT 'id',
title STRING COMMENT '标题',
city STRING COMMENT '城市',
salary STRING COMMENT '薪资',
campus_job_kind STRING COMMENT '经验',
labels STRING COMMENT '标签',
compName STRING COMMENT '公司',
compIndustry STRING COMMENT '公司领域',
compScale STRING COMMENT '公司规模'
) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE;
LOAD local DATA INPATH './data.csv' OVERWRITE INTO TABLE flask_work.work_base;
3、创建hive ads层数仓表进行分析
-- 4. 热门公司分析
CREATE TABLE top_companies (
company_name STRING COMMENT '公司名称',
job_count INT COMMENT '职位数量'
) STORED AS TEXTFILE;
INSERT INTO top_companies
SELECT compName, COUNT(*) AS job_count
FROM work_base
GROUP BY compName
ORDER BY job_count DESC
LIMIT 10;
-- 5. 岗位分布情况分析
CREATE TABLE job_distribution (
job_title STRING COMMENT '岗位名称',
job_count INT COMMENT '职位数量'
) STORED AS TEXTFILE;
INSERT INTO job_distribution
SELECT title, COUNT(*) AS job_count
FROM work_base
GROUP BY title;
-- 6. 学历要求分析
CREATE TABLE education_requirements (
education_level STRING COMMENT '学历要求',
job_count INT COMMENT '职位数量'
) STORED AS TEXTFILE;
INSERT INTO education_requirements
SELECT
CASE
WHEN labels LIKE '%博士%' THEN '博士'
WHEN labels LIKE '%硕士%' THEN '硕士'
WHEN labels LIKE '%本科%' THEN '本科'
WHEN labels LIKE '%大专%' THEN '大专'
ELSE '其他'
END AS education_level,
COUNT(*) AS job_count
FROM work_base
GROUP BY education_level;
-- 7. 薪资待遇分析(各个城市的平均薪资)
CREATE TABLE city_salary_analysis (
city STRING COMMENT '城市',
avg_salary DOUBLE COMMENT '平均薪资'
) STORED AS TEXTFILE;
INSERT INTO city_salary_analysis
SELECT city, AVG(CAST(salary AS DOUBLE)) AS avg_salary
FROM work_base
WHERE salary RLIKE '^[0-9]+$'
GROUP BY city;
5、将hive分析的结果数据导入mysql
使用sqoop迁移数据
sqoop export \
--connect jdbc:mysql://localhost:3306/flask_work \
--username root --password '123456' \
--table city_job_count \
--export-dir /hive/warehouse/flask_work.db/flask_work.city_job_count \
--input-fields-terminated-by '\001' \
--input-lines-terminated-by '\n';
sqoop export \
--connect jdbc:mysql:// localhost:3306/flask_work \
--username root --password 123456 \
--table job_salary_analysis \
--export-dir /user/hive/warehouse/flask_work.db/flask_work.job_salary_analysis \
--input-fields-terminated-by '\001' \
--input-lines-terminated-by '\n'
sqoop export \
--connect jdbc:mysql:// localhost:3306/flask_work \
--username root --password 123456 \
--table top_companies \
--export-dir /user/hive/warehouse/flask_work.db/flask_work.top_companies \
--input-fields-terminated-by '\001' \
--input-lines-terminated-by '\n'
sqoop export \
--connect jdbc:mysql:// localhost:3306/flask_work \
--username root --password 123456 \
--table job_distribution \
--export-dir /user/hive/warehouse/flask_work.db/flask_work.job_distribution \
--input-fields-terminated-by '\001' \
--input-lines-terminated-by '\n'
6. 后端服务(Flask)
使用 Flask 构建后端服务,编写rest api,读取mysql数据提供给前端页面进行展示
app.py
from flask import Flask, render_template, request, flash, redirect, url_for
from data import *
from service.task_service import get_user, register_user
app = Flask(__name__)
app.secret_key = 'b6b52fae-5618-4805-b368-501c62c6d1df'
@app.after_request
def add_header(response):
response.cache_control.max_age = 0
return response
@app.route('/', methods=['GET', 'POST'])
def login():
if request.method == 'POST':
username = request.form['username']
password = request.form['password']
user = get_user(username, password)
# 检查用户是否存在
if user is not None:
data = SourceData()
return render_template('index.html', form=data, title=data.title)
else:
# 用户名或密码错误,显示错误消息
flash('用户名或密码错误')
return redirect(url_for('login')) # 重定向回登录页面
# 如果是 GET 请求,则直接返回登录页面
return render_template('login.html')
@app.route('/register', methods=['GET', 'POST'])
def register():
if request.method == 'POST':
username = request.form.get('username')
password = request.form.get('password')
if username and password:
register_user(username, password) # 确保此函数已定义
return "注册成功!"
flash('用户名和密码不能为空')
return redirect(url_for('register'))
return render_template('register.html')
if __name__ == "__main__":
app.run(host='127.0.0.1', debug=False)
task_service.py
import pymysql
db_config = {
'host': '127.0.0.1',
'user': 'root',
'password': '12345678',
'database': 'flask_work',
'charset': 'utf8mb4',
'cursorclass': pymysql.cursors.DictCursor
}
connection = pymysql.connect(**db_config)
def get_user(username,password):
try:
with connection.cursor() as cursor:
select_query = "select * from user where username = %s and password = %s"
cursor.execute(select_query,(username,password))
result = cursor.fetchall()
return result[0]
except Exception as e:
print(e)
return None
def get_title_count():
try:
with connection.cursor() as cursor:
select_query = "select count(distinct(city)) city,count(distinct(compName)) compName from work_base;"
cursor.execute(select_query)
result = cursor.fetchall()
a=result[0]['city']
b=result[0]['compName']
return a,b
except Exception as e:
print(e)
return None
def work_count_by_city():
try:
with connection.cursor() as cursor:
select_query = "select city,job_count from city_job_count order by job_count desc limit 10"
cursor.execute(select_query)
result = cursor.fetchall()
re_list = []
for re in result:
re_list.append({"name": re['city'], "value": re['job_count']})
print(re_list)
return re_list
except Exception as e:
print(e)
return None
def work_avg_salary():
try:
with connection.cursor() as cursor:
select_query = "select job_title,avg_salary from job_salary_analysis order by avg_salary desc limit 10;"
cursor.execute(select_query)
result = cursor.fetchall()
re_list = []
for re in result:
re_list.append({"name": re['job_title'][0:8], "value": int(re['avg_salary'])})
print(re_list)
return re_list
except Exception as e:
print(e)
return None
def top_companies():
try:
with connection.cursor() as cursor:
select_query = "select company_name,job_count from top_companies limit 3;"
cursor.execute(select_query)
result = cursor.fetchall()
re_list = []
for re in result:
re_list.append({"name": re['company_name'], "value": re['job_count']})
print(re_list)
return re_list
except Exception as e:
print(e)
return None
def job_distribution_count():
try:
with connection.cursor() as cursor:
select_query = "select job_title,job_count from job_distribution order by job_count desc limit 10;"
cursor.execute(select_query)
result = cursor.fetchall()
re_list = []
for re in result:
re_list.append({"name": re['job_title'][0:6], "value": re['job_count'],"value2": 20, "color": "01", "radius": ['59%', '70%']})
print(re_list)
return re_list
except Exception as e:
print(e)
return None
def register_user(username,password):
try:
with connection.cursor() as cursor:
select_query = "insert into user(username,password) values(%s,%s)"
cursor.execute(select_query,(username,password))
connection.commit()
except Exception as e:
print(e)
return None
def education_requirements():
try:
with connection.cursor() as cursor:
select_query = "select education_level,job_count from education_requirements;"
cursor.execute(select_query)
result = cursor.fetchall()
re_list = []
for re in result:
re_list.append({"name": re['education_level'], "value": int(re['job_count'])})
return re_list
except Exception as e:
print(e)
return None
def city_salary_analysis():
try:
with connection.cursor() as cursor:
select_query = "select city,avg_salary from city_salary_analysis order by avg_salary desc limit 10;"
cursor.execute(select_query)
result = cursor.fetchall()
re_list = []
for re in result:
re_list.append({"name": re['city'], "value": int(re['avg_salary'])})
return re_list
except Exception as e:
print(e)
return None
if __name__ == '__main__':
print(city_salary_analysis())
7 页面设计
前端采用 ECharts 制作一个招聘网站大数据分析的可视化大屏。
使用 ECharts 渲染数据
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<title>index</title>
<script type="text/javascript" src="../static/js/jquery.js"></script>
<script type="text/javascript" src="../static/js/echarts.min.js"></script>
<script type="text/javascript" src="../static/js/china.js"></script>
<link rel="stylesheet" href="../static/css/comon0.css">
</head>
<script>
$(window).load(function(){
$(".loading").fadeOut()
})
/****/
$(document).ready(function(){
var whei=$(window).width()
$("html").css({fontSize:whei/20})
$(window).resize(function(){
var whei=$(window).width()
$("html").css({fontSize:whei/20})
});
});
</script>
<script type="text/javascript" src="../static/js/echarts.min.js"></script>
<script type="text/javascript" src="../static/js/china.js"></script>
<body>
<div class="canvas" style="opacity: .2">
<iframe frameborder="0" src="../static/js/index.html" style="width: 100%; height: 100%"></iframe>
</div>
<div class="loading">
<div class="loadbox"><img src="../static/picture/loading.gif"> 页面加载中...</div>
</div>
<div class="head">
<h1>{{title}}</h1>
<div class="weather">
<!-- <img src="picture/weather.png"><span>多云转小雨</span>-->
<span id="showTime"></span>
</div>
<script>
var t = null;
t = setTimeout(time,1000);//開始运行
function time()
{
clearTimeout(t);//清除定时器
dt = new Date();
var y=dt.getFullYear();
var mt=dt.getMonth()+1;
var day=dt.getDate();
var h=dt.getHours();//获取时
var m=dt.getMinutes();//获取分
var s=dt.getSeconds();//获取秒
document.getElementById("showTime").innerHTML = y+"年"+mt+"月"+day+"日"+"-"+h+"时"+m+"分"+s+"秒";
t = setTimeout(time,1000); //设定定时器,循环运行
}
</script>
</div>
<div class="mainbox">
<ul class="clearfix">
<li>
<div class="boxall" style="height: 3.2rem">
<div class="alltitle">{{form.echart1.title}}</div>
<div class="allnav" id="echart1"></div>
<div class="boxfoot"></div>
</div>
<div class="boxall" style="height: 3.2rem">
<div class="alltitle">{{form.echart2.title}}</div>
<div class="allnav" id="echart2"></div>
<div class="boxfoot"></div>
</div>
<div class="boxall" style="height: 3.2rem">
<div style="height:100%; width: 100%;">
<div class="alltitle">{{form.echart3.title}}</div>
<div class="allnav" id="echart3"></div>
</div>
<div class="boxfoot">
</div>
</div>
</li>
<li>
<div class="bar">
<div class="barbox">
<ul class="clearfix">
<li class="pulll_left counter">{{form.counter.value}}</li>
<li class="pulll_left counter">{{form.counter2.value}}</li>
</ul>
</div>
<div class="barbox2">
<ul class="clearfix">
<li class="pulll_left">{{form.counter.name}}</li>
<li class="pulll_left">{{form.counter2.name}}</li>
</ul>
</div>
</div>
<div class="map">
<div class="map1"><img src="../static/picture/lbx.png"></div>
<div class="map2"><img src="../static/picture/jt.png"></div>
<div class="map3"><img src="../static/picture/map.png"></div>
<div class="map4" id="map_1"></div>
</div>
</li>
<li>
<div class="boxall" style="height:3.4rem">
<div class="alltitle">{{form.echart4.title}}</div>
<div class="allnav" id="echart4"></div>
<div class="boxfoot"></div>
</div>
<div class="boxall" style="height: 3.2rem">
<div class="alltitle">{{form.echart5.title}}</div>
<div class="allnav" id="echart5"></div>
<div class="boxfoot"></div>
</div>
<div class="boxall" style="height: 3rem">
<div class="alltitle">{{form.echart6.title}}</div>
<div class="allnav" id="echart6"></div>
<div class="boxfoot"></div>
</div>
</li>
</ul>
</div>
<div class="back"></div>
<!--echart1-->
<script>
$(function echarts_1() {
// 基于准备好的dom,初始化echarts实例
var myChart = echarts.init(document.getElementById('echart1'));
option = {
// backgroundColor: '#00265f',
tooltip: {
trigger: 'axis',
axisPointer: {
type: 'shadow'
}
},
grid: {
left: '0%',
top:'10px',
right: '0%',
bottom: '4%',
containLabel: true
},
xAxis: [{
type: 'category',
data: {{form.echart1.xAxis|safe}},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
axisTick: {
show: false,
},
axisLabel: {
interval: 0,
// rotate:50,
show: true,
splitNumber: 15,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '12',
},
},
}],
yAxis: [{
type: 'value',
axisLabel: {
//formatter: '{value} %'
show:true,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '12',
},
},
axisTick: {
show: false,
},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1 )",
width: 1,
type: "solid"
},
},
splitLine: {
lineStyle: {
color: "rgba(255,255,255,.1)",
}
}
}],
series: [
{
type: 'bar',
data: {{form.echart1.series|safe}},
barWidth:'35%', //柱子宽度
// barGap: 1, //柱子之间间距
itemStyle: {
normal: {
color:'#2f89cf',
opacity: 1,
barBorderRadius: 5,
}
}
}
]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
window.addEventListener("resize",function(){
myChart.resize();
});
})
</script>
<!--echart2-->
<script>
$(function echarts_2() {
// 基于准备好的dom,初始化echarts实例
var myChart = echarts.init(document.getElementById('echart2'));
option = {
tooltip: {
trigger: 'item' // 修改为 'item' 以适应散点图
},
grid: {
left: '0%',
top: '10px',
right: '0%',
bottom: '4%',
containLabel: true
},
xAxis: [{
type: 'category',
data: {{form.echart2.xAxis|safe}},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
axisTick: {
show: false,
},
axisLabel: {
interval: 0,
rotate: 45, // 将标签旋转90度
show: true,
splitNumber: 15,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '8',
},
},
}],
yAxis: [{
type: 'value',
axisLabel: {
show: true,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '12',
},
},
axisTick: {
show: false,
},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
splitLine: {
lineStyle: {
color: "rgba(255,255,255,.1)",
}
}
}],
series: [{
type: 'scatter', // 将类型更改为 'scatter'
data: {{form.echart2.series|safe}},
symbolSize: 10, // 设置散点的大小
itemStyle: {
normal: {
color: '#27d08a',
opacity: 1,
}
}
}]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
window.addEventListener("resize", function() {
myChart.resize();
});
})
</script>
<!--echart3-->
<script>
$(function echarts_3() {
// 基于准备好的dom,初始化echarts实例
var myChart = echarts.init(document.getElementById('echart3'));
// 配置项
var option = {
tooltip: {
trigger: 'item',
formatter: '{a} <br/>{b}: {c} ({d}%)' // 在悬停提示框中显示公司名称、数值和百分比
},
legend: {
top: '5%',
left: 'center',
textStyle: {
color: "rgba(255,255,255,.6)"
}
},
series: [
{
name: '公司数据', // 扇形图系列名称
type: 'pie',
radius: ['40%', '70%'], // 内外半径,形成环形图
avoidLabelOverlap: false,
itemStyle: {
borderRadius: 10,
borderColor: '#fff',
borderWidth: 2
},
labelLine: {
show: true,
length: 6,
length2: 8,
lineStyle: {
color: "rgba(255,255,255,.6)"
}
},
data:{{form.echart3.data|safe}},
}
]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
window.addEventListener("resize", function() {
myChart.resize();
});
});
</script>
<!--echarts4-->
<script>
$(function echarts_4() {
// 基于准备好的dom,初始化echarts实例
var myChart = echarts.init(document.getElementById('echart4'));
option = {
tooltip: {
trigger: 'item' // 修改为 'item' 以适应散点图
},
grid: {
left: '0%',
top: '10px',
right: '0%',
bottom: '4%',
containLabel: true
},
xAxis: [{
type: 'category',
data: {{form.echart4.xAxis|safe}},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
axisTick: {
show: false,
},
axisLabel: {
interval: 0,
rotate: 45, // 将标签旋转90度
show: true,
splitNumber: 15,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '8',
},
},
}],
yAxis: [{
type: 'value',
axisLabel: {
show: true,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '12',
},
},
axisTick: {
show: false,
},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
splitLine: {
lineStyle: {
color: "rgba(255,255,255,.1)",
}
}
}],
series: [{
type: 'scatter', // 将类型更改为 'scatter'
data: {{form.echart4.series|safe}},
symbolSize: 10, // 设置散点的大小
itemStyle: {
normal: {
color: '#27d08a',
opacity: 1,
}
}
}]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
window.addEventListener("resize", function() {
myChart.resize();
});
})
</script>
<!--echarts5-->
<script>
$(function echarts_5() {
// 基于准备好的dom,初始化echarts实例
var myChart = echarts.init(document.getElementById('echart5'));
option = {
tooltip: {
trigger: 'axis',
axisPointer: {
type: 'line' // 修改为'line'以匹配折线图的样式
}
},
grid: {
left: '0%',
top: '10px',
right: '0%',
bottom: '2%',
containLabel: true
},
xAxis: [{
type: 'category',
data: {{form.echart5.xAxis|safe}},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
axisTick: {
show: false,
},
axisLabel: {
rotate: 90,
interval: 0,
show: true,
splitNumber: 15,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '12',
},
},
}],
yAxis: [{
type: 'value',
axisLabel: {
show: true,
textStyle: {
color: "rgba(255,255,255,.6)",
fontSize: '12',
},
},
axisTick: {
show: false,
},
axisLine: {
show: true,
lineStyle: {
color: "rgba(255,255,255,.1)",
width: 1,
type: "solid"
},
},
splitLine: {
lineStyle: {
color: "rgba(255,255,255,.1)",
}
}
}],
series: [{
type: 'line', // 修改为'line'类型
data: {{form.echart5.series|safe}},
smooth: true, // 可选:让折线平滑
itemStyle: {
color: '#2f89cf',
opacity: 1,
},
lineStyle: {
width: 2, // 线条宽度
type: 'solid' // 线条类型
}
}]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
window.addEventListener("resize", function() {
myChart.resize();
});
})
</script>
<!--echarts6-->
<script>
$(function echarts_6() {
var myChart = echarts.init(document.getElementById('echart6'));
var option = {
tooltip: {},
radar: {
indicator: {{form.echart6.data|safe}}
},
series: [{
type: 'radar',
data: [
{
value: [5500, 7000, 11064, 9500, 7469, 6250, 0, 7500, 6250, 7000],
name: '城市指标'
}
]
}]
};
myChart.setOption(option);
window.addEventListener("resize", function() {
myChart.resize();
});
});
</script>
<!--map_1-->
<script>
$(function map() {
// 基于准备好的dom,初始化echarts实例
var myChart = echarts.init(document.getElementById('map_1'));
var data = {{form.map_1.data|safe}};
var geoCoordMap = {
'海门':[121.15,31.89],
'鄂尔多斯':[109.781327,39.608266],
'招远':[120.38,37.35],
'舟山':[122.207216,29.985295],
'齐齐哈尔':[123.97,47.33],
'盐城':[120.13,33.38],
'赤峰':[118.87,42.28],
'青岛':[120.33,36.07],
'乳山':[121.52,36.89],
'金昌':[102.188043,38.520089],
'泉州':[118.58,24.93],
'莱西':[120.53,36.86],
'日照':[119.46,35.42],
'胶南':[119.97,35.88],
'南通':[121.05,32.08],
'拉萨':[91.11,29.97],
'云浮':[112.02,22.93],
'梅州':[116.1,24.55],
'文登':[122.05,37.2],
'上海':[121.48,31.22],
'攀枝花':[101.718637,26.582347],
'威海':[122.1,37.5],
'承德':[117.93,40.97],
'厦门':[118.1,24.46],
'汕尾':[115.375279,22.786211],
'潮州':[116.63,23.68],
'丹东':[124.37,40.13],
'太仓':[121.1,31.45],
'曲靖':[103.79,25.51],
'烟台':[121.39,37.52],
'福州':[119.3,26.08],
'瓦房店':[121.979603,39.627114],
'即墨':[120.45,36.38],
'抚顺':[123.97,41.97],
'玉溪':[102.52,24.35],
'张家口':[114.87,40.82],
'阳泉':[113.57,37.85],
'莱州':[119.942327,37.177017],
'湖州':[120.1,30.86],
'汕头':[116.69,23.39],
'昆山':[120.95,31.39],
'宁波':[121.56,29.86],
'湛江':[110.359377,21.270708],
'揭阳':[116.35,23.55],
'荣成':[122.41,37.16],
'连云港':[119.16,34.59],
'葫芦岛':[120.836932,40.711052],
'常熟':[120.74,31.64],
'东莞':[113.75,23.04],
'河源':[114.68,23.73],
'淮安':[119.15,33.5],
'泰州':[119.9,32.49],
'南宁':[108.33,22.84],
'营口':[122.18,40.65],
'惠州':[114.4,23.09],
'江阴':[120.26,31.91],
'蓬莱':[120.75,37.8],
'韶关':[113.62,24.84],
'嘉峪关':[98.289152,39.77313],
'广州':[113.23,23.16],
'延安':[109.47,36.6],
'太原':[112.53,37.87],
'清远':[113.01,23.7],
'中山':[113.38,22.52],
'昆明':[102.73,25.04],
'寿光':[118.73,36.86],
'盘锦':[122.070714,41.119997],
'长治':[113.08,36.18],
'深圳':[114.07,22.62],
'珠海':[113.52,22.3],
'宿迁':[118.3,33.96],
'咸阳':[108.72,34.36],
'铜川':[109.11,35.09],
'平度':[119.97,36.77],
'佛山':[113.11,23.05],
'海口':[110.35,20.02],
'江门':[113.06,22.61],
'章丘':[117.53,36.72],
'肇庆':[112.44,23.05],
'大连':[121.62,38.92],
'临汾':[111.5,36.08],
'吴江':[120.63,31.16],
'石嘴山':[106.39,39.04],
'沈阳':[123.38,41.8],
'苏州':[120.62,31.32],
'茂名':[110.88,21.68],
'嘉兴':[120.76,30.77],
'长春':[125.35,43.88],
'胶州':[120.03336,36.264622],
'银川':[106.27,38.47],
'张家港':[120.555821,31.875428],
'三门峡':[111.19,34.76],
'锦州':[121.15,41.13],
'南昌':[115.89,28.68],
'柳州':[109.4,24.33],
'三亚':[109.511909,18.252847],
'自贡':[104.778442,29.33903],
'吉林':[126.57,43.87],
'阳江':[111.95,21.85],
'泸州':[105.39,28.91],
'西宁':[101.74,36.56],
'宜宾':[104.56,29.77],
'呼和浩特':[111.65,40.82],
'成都':[104.06,30.67],
'大同':[113.3,40.12],
'镇江':[119.44,32.2],
'桂林':[110.28,25.29],
'张家界':[110.479191,29.117096],
'宜兴':[119.82,31.36],
'北海':[109.12,21.49],
'西安':[108.95,34.27],
'金坛':[119.56,31.74],
'东营':[118.49,37.46],
'牡丹江':[129.58,44.6],
'遵义':[106.9,27.7],
'绍兴':[120.58,30.01],
'扬州':[119.42,32.39],
'常州':[119.95,31.79],
'潍坊':[119.1,36.62],
'重庆':[106.54,29.59],
'台州':[121.420757,28.656386],
'南京':[118.78,32.04],
'滨州':[118.03,37.36],
'贵阳':[106.71,26.57],
'无锡':[120.29,31.59],
'本溪':[123.73,41.3],
'克拉玛依':[84.77,45.59],
'渭南':[109.5,34.52],
'马鞍山':[118.48,31.56],
'宝鸡':[107.15,34.38],
'焦作':[113.21,35.24],
'句容':[119.16,31.95],
'北京':[116.46,39.92],
'徐州':[117.2,34.26],
'衡水':[115.72,37.72],
'包头':[110,40.58],
'绵阳':[104.73,31.48],
'乌鲁木齐':[87.68,43.77],
'枣庄':[117.57,34.86],
'杭州':[120.19,30.26],
'淄博':[118.05,36.78],
'鞍山':[122.85,41.12],
'溧阳':[119.48,31.43],
'库尔勒':[86.06,41.68],
'安阳':[114.35,36.1],
'开封':[114.35,34.79],
'济南':[117,36.65],
'德阳':[104.37,31.13],
'温州':[120.65,28.01],
'九江':[115.97,29.71],
'邯郸':[114.47,36.6],
'临安':[119.72,30.23],
'兰州':[103.73,36.03],
'沧州':[116.83,38.33],
'临沂':[118.35,35.05],
'南充':[106.110698,30.837793],
'天津':[117.2,39.13],
'富阳':[119.95,30.07],
'泰安':[117.13,36.18],
'诸暨':[120.23,29.71],
'郑州':[113.65,34.76],
'哈尔滨':[126.63,45.75],
'聊城':[115.97,36.45],
'芜湖':[118.38,31.33],
'唐山':[118.02,39.63],
'平顶山':[113.29,33.75],
'邢台':[114.48,37.05],
'德州':[116.29,37.45],
'济宁':[116.59,35.38],
'荆州':[112.239741,30.335165],
'宜昌':[111.3,30.7],
'义乌':[120.06,29.32],
'丽水':[119.92,28.45],
'洛阳':[112.44,34.7],
'秦皇岛':[119.57,39.95],
'株洲':[113.16,27.83],
'石家庄':[114.48,38.03],
'莱芜':[117.67,36.19],
'常德':[111.69,29.05],
'保定':[115.48,38.85],
'湘潭':[112.91,27.87],
'金华':[119.64,29.12],
'岳阳':[113.09,29.37],
'长沙':[113,28.21],
'衢州':[118.88,28.97],
'廊坊':[116.7,39.53],
'菏泽':[115.480656,35.23375],
'合肥':[117.27,31.86],
'武汉':[114.31,30.52],
'大庆':[125.03,46.58]
};
var convertData = function (data) {
var res = [];
for (var i = 0; i < data.length; i++) {
var geoCoord = geoCoordMap[data[i].name];
if (geoCoord) {
res.push({
name: data[i].name,
value: geoCoord.concat(data[i].value)
});
}
}
return res;
};
option = {
tooltip : {
trigger: 'item',
formatter: function (params) {
if(typeof(params.value)[2] == "undefined"){
return params.name + ' : ' + params.value;
}else{
return params.name + ' : ' + params.value[2];
}
}
},
geo: {
map: 'china',
label: {
emphasis: {
show: false
}
},
roam: false,//禁止其放大缩小
itemStyle: {
normal: {
areaColor: '#4c60ff',
borderColor: '#002097'
},
emphasis: {
areaColor: '#293fff'
}
}
},
series : [
{
name: '消费金额',
type: 'scatter',
coordinateSystem: 'geo',
data: convertData(data),
symbolSize: function (val) {
return val[2] / {{form.map_1.symbolSize}};
},
label: {
normal: {
formatter: '{b}',
position: 'right',
show: false
},
emphasis: {
show: true
}
},
itemStyle: {
normal: {
color: '#ffeb7b'
}
}
}
]
};
myChart.setOption(option);
window.addEventListener("resize",function(){
myChart.resize();
});
}
)
</script>
</body>
</html>
8、总结
通过 request爬虫获取数据,使用Hive 进行大数据分析、通过sqoop进行数据迁移,最后使用Flask 构建后端接口获取mysql数据提供给前端,再结合 ECharts 前端可视化,能够构建一个完整的大数据展示系统。
如有遇到问题可以找小编沟通交流哦。另外小编帮忙辅导大课作业,学生毕设等。不限于MapReduce, MySQL, python,java,大数据,模型训练等。 hadoop hdfs yarn spark Django flask flink kafka flume datax sqoop seatunnel echart可视化 机器学习等