赞
踩
建一个Springboot项目。
需要做的就是 配置一个datasource,加一个mybatis-plus即可。
网上教程很多,不多说了。
如图,将资源包全部添加即可。资源在文末。
登录阿里云视频点播平台
按照需要的内容导出即可
注意:有可能出现导出之后下载不了的情况,这个事情就需要在阿里云提工单,从后台导出,会更快一些。
导出的是 .csv文件,和Excel一样的打开方式,类似下面这种文件内容。
创建表 t_video
CREATE TABLE `t_video` ( `ID` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', `MEDIA_ID` varchar(100) DEFAULT NULL COMMENT '媒体ID', `MEDIA_NAME` varchar(100) DEFAULT NULL COMMENT '媒体名称', `MEDIA_DURATION` varchar(100) DEFAULT NULL COMMENT '媒体时长(s)', `MEDIA_SIZE` varchar(100) DEFAULT NULL COMMENT '媒体大小(bytes)', `TYPE` varchar(100) DEFAULT NULL COMMENT '分类', `TYPE_CODE` varchar(100) DEFAULT NULL COMMENT '分类CODE', `MEDIA_CREATE_TIME` varchar(100) DEFAULT NULL COMMENT '创建时间', `MEDIA_UPDATE_TIME` varchar(100) DEFAULT NULL COMMENT '最新更新', `MEDIA_SOURCE_PATH` varchar(2000) DEFAULT NULL COMMENT '源文件地址', `MEDIA_MP4_PATH` varchar(2000) DEFAULT NULL COMMENT '源文件地址MP4', `MEDIA_STANDARD_PATH` varchar(2000) DEFAULT NULL COMMENT '标清_M3U8', `MEDIA_SOURCE_MP4_PATH` varchar(2000) DEFAULT NULL COMMENT '原画_MP4', `MEDIA_STANDARD_MP4_PATH` varchar(2000) DEFAULT NULL COMMENT '标清_MP4', `CREATE_TIME` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `UPDATE_TIME` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', `STATUS` char(1) DEFAULT '0' COMMENT '状态:0-未同步,1-已同步,2-数据异常,3-视频无法访问,4-无视频', `SUCCESS` varchar(50) DEFAULT '0' COMMENT '校验视频 0:未校验;1:无视频;有视频则保存视频状态', PRIMARY KEY (`ID`), KEY `NK_MEDIA_CREATE_TIME` (`MEDIA_CREATE_TIME`) USING BTREE, KEY `NK_STATUS` (`STATUS`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='视频同步表';
重要字段说明:
MEDIA_NAME:就是上传视频的唯一标识,依据该标识可以找到对应的视频信息。
TYPE:迁移之后,新的环境对应的视频分类是不一样的,所以需要做一下映射。
MEDIA_MP4_PATH:视频的访问地址,可以用该地址访问视频内容。
STATUS:用来记录视频迁移状态的字段,0-未同步,1-已同步,2-数据异常,3-视频无法访问,4-无视频
SUCCESS:用来做视频回溯校验的字段,校验视频 0:未校验;1:无视频;有视频则保存视频状态
server: port: 8008 servlet: context-path: /export spring: mvc: view: suffix: .html jackson: date-format: yyyy-MM-dd HH:mm:ss time-zone: GMT+8 servlet: multipart: max-file-size: 100MB max-request-size: 100MB # 数据源配置 datasource: username: root password: root driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://localhost:3306/video?characterEncoding=utf8&useSSL=false&useTimezone=true&serverTimezone=GMT%2B8&nullCatalogMeansCurrent=true type: com.alibaba.druid.pool.DruidDataSource initialSize: 5 min: 5 maxActive: 20 maxWait: 60000 timeBetweenEvictionRunsMillis: 60000 minEvictableIdleTimeMillis: 300000 validationQuery: SELECT 1 FROM DUAL testWhileIdle: true testOnBorrow: false testOnReturn: false poolPreparedStatements: true # 配置监控统计拦截的filters,去掉后监控界面sql无法统计,'wall'用于防火墙 filters: config,stat,wall maxPoolPreparedStatementPerConnectionSize: 20 useGlobalDataSourceStat: true connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 # 使用mybatis-plus mybatis-plus: mapper-locations: - classpath*:/mybatis/mapper/*.xml type-aliases-package: com.export.entity; global-config: id-type: 3 db-column-underline: true field-strategy: 2 configuration: log-impl: org.apache.ibatis.logging.stdout.StdOutImpl map-underscore-to-camel-case: false call-setters-on-nulls: true logging: file: /log/generate.log level: com.cpic.chjb: INFO root: INFO info: app: name: test
VideoBaseMapper.xml
<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> <mapper namespace="com.export.VideoBaseMapper"> <!-- 通用查询映射结果 --> <resultMap id="BaseResultMap" type="com.export.VideoEntity"> <id column="ID" property="id" /> <result column="MEDIA_ID" property="mediaId" /> <result column="MEDIA_NAME" property="mediaName" /> <result column="MEDIA_DURATION" property="mediaDuration" /> <result column="MEDIA_SIZE" property="mediaSize" /> <result column="TYPE" property="type" /> <result column="TYPE_CODE" property="typeCode" /> <result column="MEDIA_CREATE_TIME" property="mediaCreateTime" /> <result column="MEDIA_UPDATE_TIME" property="mediaUpdateTime" /> <result column="MEDIA_SOURCE_PATH" property="mediaSourcePath" /> <result column="MEDIA_MP4_PATH" property="mediaMp4Path" /> <result column="MEDIA_STANDARD_PATH" property="mediaStandardPath" /> <result column="MEDIA_SOURCE_MP4_PATH" property="mediaSourceMp4Path" /> <result column="MEDIA_STANDARD_MP4_PATH" property="mediaStandardMp4Path" /> <result column="CREATE_TIME" property="createTime" /> <result column="UPDATE_TIME" property="updateTime" /> <result column="STATUS" property="status" /> <result column="SUCCESS" property="success" /> </resultMap> <!-- 通用查询结果列 --> <sql id="Base_Column_List"> ID, MEDIA_ID, MEDIA_NAME, MEDIA_DURATION, MEDIA_SIZE, TYPE, TYPE_CODE, MEDIA_CREATE_TIME, MEDIA_UPDATE_TIME, MEDIA_SOURCE_PATH, MEDIA_MP4_PATH, MEDIA_STANDARD_PATH, MEDIA_SOURCE_MP4_PATH, MEDIA_STANDARD_MP4_PATH, CREATE_TIME, UPDATE_TIME, STATUS, SUCCESS </sql> </mapper>
VideoEntity.java
package com.export; import com.baomidou.mybatisplus.activerecord.Model; import com.baomidou.mybatisplus.annotations.TableId; import com.baomidou.mybatisplus.annotations.TableName; import com.baomidou.mybatisplus.enums.IdType; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import java.io.Serializable; import java.util.Date; @Data @EqualsAndHashCode(callSuper = false) @Accessors(chain = true) @TableName("T_VIDEO") public class VideoEntity extends Model<VideoEntity> { private static final long serialVersionUID = 1L; /** * 自增列 */ @TableId(value = "id", type = IdType.AUTO) private Integer id; //媒体ID private String mediaId; //媒体名称 private String mediaName; //媒体时长(s) private String mediaDuration; //媒体大小(bytes) private String mediaSize; //分类 private String type; //分类CODE private String typeCode; //创建时间 private String mediaCreateTime; //最新更新 private String mediaUpdateTime; //源文件地址 private String mediaSourcePath; //源文件地址MP4 private String mediaMp4Path; //标清_M3U8 private String mediaStandardPath; //原画_MP4 private String mediaSourceMp4Path; //标清_MP4 private String mediaStandardMp4Path; //创建时间 private Date createTime; //更新时间 private Date updateTime; //状态:0-未同步,1-已同步,2-数据异常,3-视频无法访问,4-无视频 private String status; //校验视频 0:未校验;1:无视频;有视频则保存视频状态 private String success; @Override protected Serializable pkVal() { return this.id; } }
VideoBaseMapper.java
package com.export;
import com.baomidou.mybatisplus.mapper.BaseMapper;
public interface VideoBaseMapper extends BaseMapper<VideoEntity> {
}
package com.export; import com.aliyun.vod.upload.impl.UploadVideoImpl; import com.aliyun.vod.upload.req.UploadStreamRequest; import com.aliyun.vod.upload.resp.UploadStreamResponse; import lombok.extern.slf4j.Slf4j; import java.io.*; import java.net.URL; import java.util.HashMap; import java.util.Map; /** * 使用上传SDK进行视频文件上传 */ @Slf4j public class VideoSynchronize { public final static String KEY_ID = "**********"; public final static String KEY_SECRET = "**********"; //视频类别映射,由原视频类型,映射成新环境视频类型 public static final Map<String,String> map = new HashMap<>(); static { map.put("11111","22222"); } /** * 流式上传接口 * @param accessKeyId KEY_ID * @param accessKeySecret KEY_SECRET * @param title 媒体名称 * @param fileName 文件名称 * @param inputStream 流 * @param cateId 文件类别 */ private static void uploadStream(String accessKeyId, String accessKeySecret, String title, String fileName, InputStream inputStream, Long cateId) throws RuntimeException{ UploadStreamRequest request = new UploadStreamRequest(accessKeyId, accessKeySecret, title, fileName, inputStream); /* 自定义消息回调设置,参数说明请参见基本数据类型 */ //request.setUserData(""{\"Extend\":{\"test\":\"www\",\"localId\":\"xxxx\"},\"MessageCallback\":{\"CallbackURL\":\"http://example.aliyundoc.com\"}}""); /* 视频分类ID(可选) */ request.setCateId(cateId); /* 视频标签,多个用逗号分隔(可选) */ //request.setTags("标签1,标签2"); /* 视频描述(可选) */ //request.setDescription("视频描述"); /* 封面图片(可选) ,如http://****.example.com/image_01.jpg*/ //request.setCoverURL("<Your CoverURL>"); /* 模板组ID(可选) */ //request.setTemplateGroupId("8c4792cbc8694e****fd5330e56a33d"); /* 工作流ID(可选) */ //request.setWorkflowId("d4430d07361f****1339577859b0177b"); /* 存储区域(可选) */ //request.setStorageLocation("outin-20170323****266-5sejdln9o.oss-cn-shanghai.aliyuncs.com"); /* 点播服务接入点 */ request.setApiRegionId("cn-shanghai"); /* ECS部署区域*/ // request.setEcsRegionId("cn-shanghai"); UploadVideoImpl uploader = new UploadVideoImpl(); UploadStreamResponse response = uploader.uploadStream(request); log.info("RequestId=" + response.getRequestId() + "\n"); //请求视频点播服务的请求ID if (response.isSuccess()) { log.info("VideoId=" + response.getVideoId() + "\n"); } else { //如果设置回调URL无效,不影响视频上传,可以返回VideoId同时会返回错误码。其他情况上传失败时,VideoId为空,此时需要根据返回错误码分析具体错误原因 log.info("VideoId=" + response.getVideoId() + "\n"); log.info("ErrorCode=" + response.getCode() + "\n"); log.info("ErrorMessage=" + response.getMessage() + "\n"); throw new RuntimeException(); } } public static void synchronizeData(VideoEntity videoBean) throws IOException { /*//用户可自行添加url数据源,并传入视频媒资信息,上传视频资源 InputStream inputStream = null; //您的视频地址。如http://example.aliyundoc.com/video*/ String url = videoBean.getMediaMp4Path(); InputStream inputStream = new URL(url).openStream(); //以下参数重的AccessKey ID, AccessKey Secret提前准备好的AccessKey信息。<Your Video Title>为视频标题。<Your Video with File Extension>为含文件扩展名的视频,如video-1.mp4。 uploadStream(KEY_ID, KEY_SECRET, videoBean.getMediaName(), videoBean.getMediaId() + ".MP4", inputStream, Long.valueOf(map.get(videoBean.getTypeCode()))); } }
package com.export; import com.baomidou.mybatisplus.mapper.EntityWrapper; import com.baomidou.mybatisplus.service.impl.ServiceImpl; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import javax.servlet.http.HttpServletResponse; import java.io.InputStream; import java.net.URL; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @Slf4j @Controller public class VideoController extends ServiceImpl<VideoBaseMapper, VideoEntity>{ private final static String VIDEO_STATUS_0 = "0";//未同步 private final static String VIDEO_STATUS_1 = "1";//已同步 private final static String VIDEO_STATUS_2 = "2";//数据异常 private final static String VIDEO_STATUS_3 = "3";//视频无法访问 private final static String VIDEO_STATUS_4 = "4";//无视频 // http://localhost:8008/export/video @RequestMapping("/video") @ResponseBody public void queryDictionaryAllList(HttpServletResponse response) throws Exception { response.setHeader("Access-Control-Allow-Origin","*"); response.setHeader("Catch-Control","no-cache"); //1、查询数据库没有执行数据 EntityWrapper<VideoEntity> wrapper = new EntityWrapper<>(); wrapper.in("STATUS", Collections.singletonList(VIDEO_STATUS_0)); wrapper.orderBy("ID", true); wrapper.isNotNull("MEDIA_MP4_PATH"); wrapper.last("limit 1000"); List<VideoEntity> videoEntityList = baseMapper.selectList(wrapper); if(null == videoEntityList || videoEntityList.size() == 0) return; //2、设置一个值放置别的线程拉取到这部分数据 for (VideoEntity video : videoEntityList) { video.setStatus("a"); } updateBatchById(videoEntityList); //3、开启一个异步线程跑数据 Date now = new Date(); Thread thread = new Thread(() ->{ for(VideoEntity videoEntity : videoEntityList){ VideoEntity videoCheck = baseMapper.selectById(videoEntity.getId()); if(null == videoCheck || VIDEO_STATUS_1.equals(videoCheck.getStatus())){ continue; } //单独校验一次文件流是否存在 try { InputStream inputStream = new URL(videoEntity.getMediaMp4Path()).openStream(); } catch (Exception e){ videoEntity.setStatus(VIDEO_STATUS_3); videoEntity.setUpdateTime(now); baseMapper.updateById(videoEntity); continue; } //同步视频 try { VideoSynchronize.synchronizeData(videoEntity); videoEntity.setStatus(VIDEO_STATUS_1); } catch (Exception e){ log.error("Current Thread Write Excel Error", e); videoEntity.setStatus(VIDEO_STATUS_2); } finally { videoEntity.setUpdateTime(now); baseMapper.updateById(videoEntity); } } // writeDataPool(videoEntityList); }); thread.start(); } //线程池(在使用线程池多线程跑视频迁移的过程中,发现存在大量迁移失败的情况,猜测应该是做了限流之类的东西) private void writeDataPool(List<VideoEntity> list) { Integer poolSize = 1000; ExecutorService pool = new ThreadPoolExecutor(poolSize, 2 * poolSize, poolSize, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(1), Executors.defaultThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy()); for (VideoEntity video : list) { pool.execute(new writeDataThread(video)); } try { pool.shutdown(); boolean loop = true; do { loop = !pool.awaitTermination(200, TimeUnit.MILLISECONDS); } while (loop); } catch (InterruptedException e) { log.error("Current Thread Pool Error", e); } } private class writeDataThread implements Runnable { private VideoEntity videoEntity; public writeDataThread(VideoEntity videoEntity) { this.videoEntity = videoEntity; } @Override public void run() { try { VideoSynchronize.synchronizeData(videoEntity); videoEntity.setStatus(VIDEO_STATUS_1); } catch (Exception e){ log.error("Current Thread Write Excel Error", e); videoEntity.setStatus(VIDEO_STATUS_2); } finally { videoEntity.setUpdateTime(new Date()); baseMapper.updateById(videoEntity); } } } }
package com.export; import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.profile.DefaultProfile; import com.aliyuncs.vod.model.v20170321.SearchMediaRequest; import com.aliyuncs.vod.model.v20170321.SearchMediaResponse; import com.baomidou.mybatisplus.mapper.EntityWrapper; import com.baomidou.mybatisplus.service.impl.ServiceImpl; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; @Slf4j @Controller public class CheckController extends ServiceImpl<VideoBaseMapper, VideoEntity> { private final static String VIDEO_SUCCESS_0 = "0";//未校验 private final static String VIDEO_SUCCESS_1 = "1";//无视频 private final static String VIDEO_SUCCESS_EXCEPTION = "Exception";//异常 /** * 根据分类和视频名称查询视频 */ @RequestMapping("/check") @ResponseBody public void queryVideoByCate() throws Exception { //1、查询数据库未校验的视频,以及校验异常的视频 EntityWrapper<VideoEntity> wrapper = new EntityWrapper<>(); wrapper.eq("STATUS", "1"); wrapper.eq("SUCCESS", VIDEO_SUCCESS_0); // wrapper.eq("SUCCESS", VIDEO_SUCCESS_EXCEPTION); List<VideoEntity> videoEntityList = baseMapper.selectList(wrapper); if (null == videoEntityList || videoEntityList.size() == 0) return; Date now = new Date(); Thread thread = new Thread(() ->{ for (VideoEntity videoEntity : videoEntityList) { try { List<VideoResponseDTO> list = checkVideo(videoEntity); if (null != list) { VideoResponseDTO videoResponseDTO = list.get(0); videoEntity.setSuccess(videoResponseDTO.getStatus()); } else { videoEntity.setSuccess(VIDEO_SUCCESS_1);//无视频 } } catch (Exception e) { videoEntity.setSuccess(VIDEO_SUCCESS_EXCEPTION);//有异常 } finally { videoEntity.setUpdateTime(now); } baseMapper.updateById(videoEntity); } }); thread.start(); } private List<VideoResponseDTO> checkVideo(VideoEntity videoEntity) throws Exception { List<VideoResponseDTO> list = new ArrayList<>(); // 拼接match条件(cateId) String matchCondition = "Status in ('UploadSucc','Transcoding','Checking','Blocked','Normal')"; //查询cateId String cateId = VideoSynchronize.map.get(videoEntity.getTypeCode()); matchCondition = matchCondition + " And CateId = " + cateId; VideoResponseDTO videoResponseDTO; if (null != videoEntity.getMediaName()) { matchCondition = matchCondition + " And Title = '" + videoEntity.getMediaName() + "'"; // 查询对应分类下所有视频 SearchMediaResponse videos = getVideos(matchCondition); for (SearchMediaResponse.Media media : videos.getMediaList()) { videoResponseDTO = new VideoResponseDTO(); SearchMediaResponse.Media.Video videoInfo = media.getVideo(); if (null != videoInfo.getVideoId()) { videoResponseDTO.setVideoId(videoInfo.getVideoId()); videoResponseDTO.setCoverURL(videoInfo.getCoverURL()); videoResponseDTO.setStatus(videoInfo.getStatus()); // 时间转化 if (null != videoInfo.getDuration()) { String s = String.valueOf(videoInfo.getDuration()); videoResponseDTO.setDuration(getDate(Integer.parseInt(s.substring(0, s.indexOf("."))))); } } SearchMediaResponse.Media.Audio audio = media.getAudio(); if (null != audio.getAudioId()) { videoResponseDTO.setVideoId(audio.getAudioId()); videoResponseDTO.setCoverURL(audio.getCoverURL()); videoResponseDTO.setStatus(audio.getStatus()); // 时间转化 if (null != audio.getDuration()) { String s = String.valueOf(audio.getDuration()); videoResponseDTO.setDuration(getDate(Integer.parseInt(s.substring(0, s.indexOf("."))))); } } list.add(videoResponseDTO); } } return list; } /** * 获取视频信息 */ public SearchMediaResponse getVideos(String matchCondition) throws Exception { DefaultAcsClient client = initVodClient(); SearchMediaRequest request = new SearchMediaRequest(); request.setFields("Title,CoverURL,Status,CreationTime,ModificationTime,VideoId,Duration"); request.setMatch(matchCondition); request.setSearchType("video"); request.setSortBy("CreationTime:Asc"); return searchMedia(client, request); } public static DefaultAcsClient initVodClient() throws IOException { String regionId = "cn-shanghai"; // 点播服务接入区域 DefaultProfile profile; profile = DefaultProfile.getProfile(regionId, VideoSynchronize.KEY_ID, VideoSynchronize.KEY_SECRET); DefaultAcsClient client = new DefaultAcsClient(profile); return client; } /*查找视频*/ public static SearchMediaResponse searchMedia(DefaultAcsClient client, SearchMediaRequest request) throws Exception { return client.getAcsResponse(request); } /** * 秒转化为时分秒 */ public static String getDate(Integer date) { if (date < 60) { if (date < 10){ return "00:0" + date; } else { return "00:" + date; } } else if (date > 60 && date < 3600) { String mm; String ss; int m = date/60; int s = date%60; if (m < 10){ mm = "0" + m; } else { mm = String.valueOf(m); } if (s < 10){ ss = "0" + s; } else { ss = String.valueOf(s); } return mm + ":" + ss; } else { String hh; String mm; String ss; int h = date/3600; int m = (date%3600)/60; int s = (date%3600)%60; if (m < 10){ mm = "0" + m; } else { mm = String.valueOf(m); } if(s < 10){ ss = "0" + s; } else { ss = String.valueOf(s); } if(h < 10){ hh = "0" + h; } else { hh = String.valueOf(h); } return hh + ":" + mm + ":" + ss; } } public class VideoResponseDTO{ //视频封面URL。 private String coverURL; //视频创建时间。使用UTC时间。 private String creationTime; //视频状态 private String status; //视频分类名称 private String cateName; //视频ID private String videoId; //视频标题 private String title; //持续时间 private String duration; public String getCoverURL() { return coverURL; } public void setCoverURL(String coverURL) { this.coverURL = coverURL; } public String getCreationTime() { return creationTime; } public void setCreationTime(String creationTime) { this.creationTime = creationTime; } public String getStatus() { return status; } public void setStatus(String status) { this.status = status; } public String getCateName() { return cateName; } public void setCateName(String cateName) { this.cateName = cateName; } public String getVideoId() { return videoId; } public void setVideoId(String videoId) { this.videoId = videoId; } public String getTitle() { return title; } public void setTitle(String title) { this.title = title; } public String getDuration() { return duration; } public void setDuration(String duration) { this.duration = duration; } } }
CSDN资源链接:https://download.csdn.net/download/qq_38254635/87353068
百度网盘资源链接:https://pan.baidu.com/s/1fcPqK449h6NYzC-ADvMZMQ?pwd=54rp
提取码: 54rp
有什么不对的还望指正,书写不易,觉得有帮助就点个赞吧!
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。