毕设项目分享 基于Spark网易云音乐数据分析_基于spark的音乐数据分析

---

0 项目说明

基于Spark网易云音乐数据分析

提示：适合用于课程设计或毕业设计，工作量达标，源码开放

---

1 系统模块

包含爬虫,Scala代码,Spark,Hadoop,ElasticSearch,logstash,Flume,echarts,log4j

emotional_analysis_spider 爬虫模块
emotional_analysis_web 数据处理模块(Scala代码)
emotional_analysis_recommend 推荐模块目前还未开发
emotional_analysis_web 报表展现模块

Crawler-sample-data 爬虫抓取样例数据
MySQL-yuncun MySQL数据库结构及数据
config ES logStash Flume 相关配置文件
mlib-text 机器学习分类测试相关数据
statistical-static-txt SparkSQL相关关联静态数据

2 分析内容

1.图计算
2.机器学习预测歌曲分类
3.评论词云
4.评论时间段
5.评论top榜
6.热歌top榜
7.用户性别比例
8.用户星座比例
9.用户年龄比例
10.用户全国地理分布
11.热评搜索等

3 界面展示

4 项目源码

package org.bianqi.wangyi.web.es.test;

import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.bianqi.web.wangyi.utils.ESClientUtils;
import org.bianqi.web.wangyi.utils.PageBean;
import org.bianqi.web.wangyi.utils.SearchUtils;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.highlight.HighlightBuilder;
import org.elasticsearch.search.highlight.HighlightField;
import org.junit.Before;
import org.junit.Test;

import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;

/**
 * ES客户端
 * <p>Title: ESClientTest</p>
 * <p>Description: </p>
 * <p>School: qiqihar university</p> 
 * @author	BQ
 * @date	2018年2月23日下午6:46:19
 * @version 1.0.0
 */
public class ESClientTest {

	private TransportClient client;
	private IndexRequest source;

	@Before
	public void before11() throws Exception {
		Settings settings = Settings.settingsBuilder().put("client.transport.ignore_cluster_name", true) // 忽略集群名字验证,
				.build();
		client = TransportClient.builder().settings(settings).build()
				.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress("192.168.254.215", 9300)));
		System.out.println("success connect");
	}
	/**
     * 查看集群信息
     */
    @Test
    public void testInfo() {
        List<DiscoveryNode> nodes = client.connectedNodes();
        for (DiscoveryNode node : nodes) {
            System.out.println(node.getHostAddress());
        }
    }
    
    
    /**
     * 组织json串, 方式1,直接拼接
     */
    public String createJson1() {
        String json = "{" +
                "\"user\":\"kimchy\"," +
                "\"postDate\":\"2013-01-30\"," +
                "\"message\":\"trying out Elasticsearch\"" +
            "}";
        return json;
    }
    
    /**
     * 使用map创建json
     */
    public Map<String, Object> createJson2() {
        Map<String,Object> json = new HashMap<String, Object>();
        json.put("user", "kimchy");
        json.put("postDate", new Date());
        json.put("message", "trying out elasticsearch");
        return json;
    }

    
    /**
     * 使用es的帮助类
     */
    public XContentBuilder createJson4() throws Exception {
        // 创建json对象, 其中一个创建json的方式
        XContentBuilder source = XContentFactory.jsonBuilder()
            .startObject()
                .field("user", "kimchy")
                .field("postDate", new Date())
                .field("message", "trying to out ElasticSearch")
            .endObject();
        return source;
    }

    @Test
    public void test1() throws Exception {
        XContentBuilder source = createJson4();
        // 存json入索引中
        IndexResponse response = client.prepareIndex("twitter", "tweet", "1").setSource(source).get();
//        // 结果获取
        String index = response.getIndex();
        String type = response.getType();
        String id = response.getId();
        long version = response.getVersion();
        boolean created = response.isCreated();
        System.out.println(index + " : " + type + ": " + id + ": " + version + ": " + created);
    }
    
    /**
     * get API 获取指定文档信息
     */
    @Test
    public void testGet() {
//        GetResponse response = client.prepareGet("twitter", "tweet", "1")
//                                .get();
        GetResponse response = client.prepareGet("log4j-2018.02.19", "log4j_type", "AWGvky2aWyh2zXt6_1Um")
                .setOperationThreaded(false)    // 线程安全
                .get();
        System.out.println(response.getSourceAsString());
    }
    
    @Test
    public void testGet1(){
    	SearchResponse response = client.prepareSearch("log4j-2018.02.19")
  			  .setTypes("log4j_type")
  			  // 设置查询类型
  			// 1.SearchType.DFS_QUERY_THEN_FETCH = 精确查询
  			// 2.SearchType.SCAN = 扫描查询,无序
  			// 3.SearchType.COUNT = 不设置的话,这个为默认值,还有的自己去试试吧
  			  .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
  			  // 设置查询关键词
  			  .setQuery(QueryBuilders.matchQuery("log4j_type.thread", "main"))
  			  .addHighlightedField("main")
  			  .setHighlighterPreTags("<em>")
  			  .setHighlighterPostTags("</em>")
  			  // 设置查询数据的位置,分页用
  			.setFrom(0)
  			// 设置查询结果集的最大条数
  			.setSize(60)
  			// 设置是否按查询匹配度排序
  			.setExplain(true)
  			// 最后就是返回搜索响应信息
  			  .execute()
  			  .actionGet();
  			SearchHits searchHits = response.getHits();
  			System.out.println("-----------------在["+"ma"+"]中搜索关键字["+"main"+"]---------------------");
  			System.out.println("共匹配到:"+searchHits.getTotalHits()+"条记录!");
  			SearchHit[] hits = searchHits.getHits();
  			for (SearchHit searchHit : hits) {
  			//获取高亮的字段
  			Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
  			HighlightField highlightField = highlightFields.get("main");
  			System.out.println("高亮字段:"+highlightField.getName()+"\n高亮部分内容:"+highlightField.getFragments()[0].string());
  			Map<String, Object> sourceAsMap = searchHit.sourceAsMap();
  			Set<String> keySet = sourceAsMap.keySet();
  			for (String string : keySet) {
  			//key value 值对应关系
  			System.out.println(string+":"+sourceAsMap.get(string));
  			}
  		}
    }
    
    @Test
    public void test4(){
    	SearchRequestBuilder responsebuilder = client.prepareSearch("twitter").setTypes("tweet") ;
    	SearchResponse myresponse=responsebuilder.setQuery(QueryBuilders.matchPhraseQuery("user", "kimchy"))  
    			.setFrom(0).setSize(10).setExplain(true).execute().actionGet();
    	SearchHits hits = myresponse.getHits();  
    	for (int i = 0; i < hits.getHits().length; i++) {  
    	           System.out.println(hits.getHits()[i].getSourceAsString());}  
     }
    
    
    @Test
    public void test5(){
        BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();  
        //使用should实现或者查询  
        boolBuilder.should(QueryBuilders.matchQuery("content","爱我"));  
        //c查询  
        SearchRequestBuilder searchRequestBuilder = this.client.prepareSearch("hotcomments")  
                .setTypes("logs")  
                .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) //设置查询类型：1.SearchType.DFS_QUERY_THEN_FETCH 精确查询； 2.SearchType.SCAN 扫描查询,无序  
                .setQuery(boolBuilder)  
                .setSize(10);  
   
        //设置高亮显示  
        searchRequestBuilder.setHighlighterPostTags("</span>");
        searchRequestBuilder.setHighlighterPreTags("<span style=\"color:red\">");
        searchRequestBuilder.addHighlightedField("content"); 
        //执行结果  
        SearchResponse response = searchRequestBuilder.get();  
        //接受结果  
        List<Map<String,Object>> result = new ArrayList<>();  
        //遍历结果  
        for(SearchHit hit:response.getHits()){  
            Map<String, Object> source = hit.getSource();  
            //处理高亮片段  
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();  
            HighlightField nameField = highlightFields.get("content");  
            if(nameField!=null){  
                Text[] fragments = nameField.fragments();  
                String nameTmp ="";  
                for(Text text:fragments){  
                    nameTmp+=text;  
                }  
                //将高亮片段组装到结果中去  
                source.put("content",nameTmp);
            }  
            result.add(source);  
        }  
        for (Map<String, Object> map : result) {
			Set<Entry<String, Object>> entrySet = map.entrySet();
			for (Entry<String, Object> entry : entrySet) {
				System.out.println(entry.getKey()+":::"+entry.getValue());
			}
		}
    }
    
    @Test
    public void test6() throws Exception{
    	PageBean<Map<String,Object>> searchListPageBean = new PageBean<Map<String,Object>>();
	    BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();  
	    boolBuilder.must(QueryBuilders.matchQuery("content","那就这样吧"));  
	    PageHelper.startPage(5, 3);
	    SearchRequestBuilder searchRequestBuilder = ESClientUtils.getClient().prepareSearch("hotcomments")  
	               .setTypes("logs")  
	               .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) //设置查询类型：1.SearchType.DFS_QUERY_THEN_FETCH 精确查询； 2.SearchType.SCAN 扫描查询,无序  
	               .setQuery(boolBuilder)  
	               .setFrom(5)  
	               .setSize(3);
	    long totalHits = ESClientUtils.getClient().prepareSearch("hotcomments")  
	            .setTypes("logs")  
	            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
	            .setQuery(boolBuilder)
	            .get()
	            .getHits().getTotalHits();
	       //设置高亮显示  
	       searchRequestBuilder.setHighlighterPostTags("</span>");
	       searchRequestBuilder.setHighlighterPreTags("<span style=\"color:red\">");
	       searchRequestBuilder.addHighlightedField("content");   
	       //执行结果  
	       SearchResponse response = searchRequestBuilder.get();  
	       //接受结果  
	       List<Map<String,Object>> result = new ArrayList<>();  
	       //遍历结果  
	       for(SearchHit hit:response.getHits()){  
	           Map<String, Object> source = hit.getSource(); 
	           String songName = null;
	           String songUrl = null;
	           Set<Entry<String, Object>> entrySet = source.entrySet();
	           for (Entry<String, Object> entry : entrySet) {
	        	   if(entry.getKey().equals("songId")){
	        		   Integer songId = (Integer)entry.getValue();
	        		   songName = SearchUtils.getSongNameById(songId);
	        		   songUrl = SearchUtils.getSongUrlById(songId);
	        	   }
	           }
	           source.put("songName", songName);
	           source.put("songUrl", songUrl);
	           //处理高亮片段  
	           Map<String, HighlightField> highlightFields = hit.getHighlightFields();  
	           HighlightField nameField = highlightFields.get("content");  
	           if(nameField!=null){  
	               Text[] fragments = nameField.fragments();  
	               String nameTmp ="";  
	               for(Text text:fragments){  
	                   nameTmp+=text;  
	               }  
	               //将高亮片段组装到结果中去  
	               source.put("content",nameTmp);  
	           }  
	           result.add(source);  
	       }  
	       PageInfo<Map<String,Object>> pageInfo = new PageInfo<>(result);
	        long totalPage = (totalHits + 3 - 1) / 3;
			int pageNum = pageInfo.getPageNum();
			searchListPageBean.setPage(pageNum);
			searchListPageBean.setTotalPage(totalPage);
			searchListPageBean.setList(result);
			
			 for (Map<String, Object> map : result) {
					Set<Entry<String, Object>> entrySet = map.entrySet();
					for (Entry<String, Object> entry : entrySet) {
						System.out.println(entry.getKey()+":::"+entry.getValue());
					}
				}
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307

5 最后

**项目分享: ** https://gitee.com/asoonis/htw