devbox
你好赵伟
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

利用java多线程爬取高清图片网站wallhaven的图片_java获取网页图片

作者：你好赵伟 | 2024-04-17 00:45:36

赞

踩

java获取网页图片

个人爬了100G高清图片 ， 用来玩玩挺好,哈哈

 

准备工作： 

1.分析网站页面以及请求

 

 获取请求网址以及参数 可知查询参数为q: id :xxx    页面page

 

 

 

 所以后期我们只需要拼接路径就可以自动获取每页的数据

"https://wallhaven.cc/search?q=id:"+类型+"&page="+页码;

分析列表页面找到 当前页面是否隐藏下载url，没有的话还需跳转到详情页面再获取下载链接

 

没有直接下载地址只有缩略图url https://th.wallhaven.cc/small/g7/g7qyzd.jpg

打开详情页面

 

找到图片原图地址 ： https://w.wallhaven.cc/full/g7/wallhaven-g7qyzd.jpg

比较发现，缩略图与原图都是这种规律

https://th.wallhaven.cc/small/g7/g7qyzd.jpg

https://w.wallhaven.cc/full/g7/wallhaven-g7qyzd.jpg

所以我们只需要根据缩略图地址就可以拼接出原图地址

所以：

先获取每页图片列表的缩略图地址

  1.  
  2.     /**
  3.      * 获取页面html,
  4.      */
  5.     public static String getHtml(String url) throws IOException {
  6.         String name = Thread.currentThread().getName();
  7.  
  8.         BufferedReader in = null;
  9.         String result="";
  10.         try (InputStream ins =  getInputStream(url)) {
  11.  
  12.             InputStreamReader read = new InputStreamReader( ins, "gb2312");// 考虑到编码格
  13.             BufferedReader bufferedReader = new BufferedReader(read);
  14.             String lineTxt = null;
  15.             //读取一行
  16.             while ((lineTxt = bufferedReader.readLine()) != null) {
  17.                 //正则表达式
  18.                 result += lineTxt;
  19.             }
  20.             read.close();
  21.             bufferedReader.close();
  22.         } catch (IOException e) {
  23.             System.out.println("线程"+name+"，页html文件不存在: " + url);
  24.            // e.printStackTrace();
  25.         }
  26.  
  27.         return result;
  28.     }
  29.  
  30.  
  31.  /**
  32.      * 获取html所有某个相同标签上属性值
  33.      * @param html
  34.      * @param element  标签元素
  35.      * @param attr 属性
  36.      * @return
  37.      */
  38.     public static List match(String html, String element, String attr) {
  39.  
  40.         List result = new ArrayList();
  41.  
  42.         String reg = "]*?\\s" + attr + "=['\"]?(.*?)['\"]?(\\s.*?)?>";
  43.  
  44.         Matcher m = Pattern.compile(reg).matcher(html);
  45.  
  46.         while (m.find()) {
  47.  
  48.             String r = m.group(1);
  49.  
  50.             result.add(r);
  51.  
  52.         }
  53.  
  54.         return result;
  55.  
  56.     }
  57.  
  58.  
  59.  
  60.  
  61.  
  62.  public static String  download(List<String> links,String type , int pageNum){
  63.         /**
  64.          https://th.wallhaven.cc/small/4l/4ly2o2.jpg
  66.          https://w.wallhaven.cc/full/4l/wallhaven-4ly2o2.jpg
  67.          */
  68.         for (String url : links  ) {
  69.             String realUrl = "https://w.wallhaven.cc/full/".concat(url.split("/")[4]) .concat("/wallhaven-".concat(url.split("/")[5]));
  70.             //System.out.println("当前路径: "+ realUrl);
  71.             String dirSec = url.split("/")[4];
  72.             String fileName = "-wallhaven-"+dirSec+"-"+url.split("/")[5];
  73.             downFiles(realUrl, type , fileName,pageNum);
  74.         }
  75.  
  76.  
  77.         return  "分任务下载完成";
  78.     }
  79.  
  80.  /**
  81.      * 下载文件
  82.      * @return
  83.      */
  84.     public static   void downFiles(String url,String type,String fileName,int pageNum){
  85.  
  86.         String name = Thread.currentThread().getName();
  87.         try (InputStream ins =  getInputStream(url)) {
  88.             if(ins.available() ==0){
  89.                 return;
  90.             }
  91.             sumNum++;
  92.             String saveDir = "I:\\picture\\"+type;
  93.             Path target = Paths.get(saveDir, sumNum+fileName);
  94.             Files.createDirectories(target.getParent());
  95.             Files.copy(ins, target, StandardCopyOption.REPLACE_EXISTING);
  96.             System.out.println("第"+pageNum+"页，"+"第"+sumNum+"张图片下载完成！");
  97.         } catch (IOException e) {
  98.             System.out.println("线程"+name+"，第"+pageNum+"页，"+"下载文件时文件不存在: "+url);
  99.            // e.printStackTrace();
  100.         }
  101.     }
  102.  
  103.     public static InputStream   getInputStream(String url) throws IOException {
  104.         URL realUrl = new URL(url);
  105.  
  106.         // 打开和URL之间的连接
  107.  
  108.         URLConnection connection = realUrl.openConnection();
  109.  
  110.         //模拟浏览器通用的请求属性
  111.  
  112.         connection.setRequestProperty("accept", "*/*");
  113.  
  114.         connection.setRequestProperty("connection", "Keep-Alive");
  115.  
  116.         connection.setRequestProperty("user-agent",
  117.  
  118.                 "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
  119.  
  120.         // 建立实际的连接
  121.  
  122.         connection.connect();
  123.  
  124.         InputStream ins = connection.getInputStream();
  125.         return ins;
  126.     }

以上就是基本思路

下面完整代码，采用线程池异步任务，由于网站有连接数限制，最大3个线程同时下载访问

  1. 异步任务配置
  2. /**
  3.  * 线程池的配置：
  4.  *  @EnableAsync 开启异步的支持
  5.  *
  6.  */
  7. @Slf4j
  8. @Configuration
  9. @ComponentScan("com.zhyqin.service")
  10. @EnableAsync
  11. public class TaskExecutorConfig implements AsyncConfigurer {
  12.     @Override
  13.     public Executor getAsyncExecutor() {
  14.         //初始化线程池
  15.         ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
  16.         taskExecutor.setMaxPoolSize(3);
  17.         taskExecutor.setCorePoolSize(3);
  18.         taskExecutor.setQueueCapacity(3);
  19.         taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
  20.         taskExecutor.initialize();
  21.         log.info("线程池初始化成功！！");
  22.         return taskExecutor;
  23.     }

异步任务：

  1. @Slf4j
  2. @Service
  3. public class AsyncTaskService {
  4.  
  5.     @Async
  6.     public void executeAsyncTask(String type, int pageNum){
  7.         String name = Thread.currentThread().getName();
  8.  
  9.         log.info("当前线程"+name+"执行任务:{},当前页码{} ",type, pageNum);
  10.         String url = "https://wallhaven.cc/search?q=id:"+type+"&page="+pageNum;
  11.         String html = null;
  12.         String downloadStatus="";
  13.         try {
  14.             html = getHtml(url);
  15.             List links = match(html,"img","data-src");
  16.             if (links==null || links.size()==0){
  17.                 Thread.sleep(5000);
  18.                 StaticMethod.failTimes = StaticMethod.failTimes+1;
  19.                 log.info("******失败次数：{}*****", StaticMethod.failTimes);
  20.                 return;
  21.             }
  22.             downloadStatus = StaticMethod.download(links,type, pageNum);
  23.  
  24.         } catch (Exception e) {
  25.             log.info("################IO执行异常#########################");
  26.  
  27.         }
  28.  
  29.  
  30.         log.info("当前线程"+name+"； 执行第{}页,执行状态：{}", pageNum, downloadStatus);
  31.     }
  32.  
  33.  
  34.  
  35.  
  36. }

由于不确定规则是否通用，错略失败40次不再下载

  1. @Slf4j
  2. @Service
  3. public class PictureService {
  4.     @Resource
  5.     AsyncTaskService asyncTaskService;
  6.  
  7.     public String startDown( String type,int startPage)  {
  8.  
  9.  
  10.         while (StaticMethod.failTimes <40){
  11.             if(startPage > 0) StaticMethod.currPageNum = startPage;
  12.  
  13.             asyncTaskService.executeAsyncTask(type,startPage);
  14.             StaticMethod.currPageNum= StaticMethod.currPageNum + 1;
  15.             startPage++;
  16.             StaticMethod.failTimes=0;
  17.             try {
  18.                 Thread.sleep(15000);
  19.             } catch (InterruptedException e) {
  20.                 throw new RuntimeException(e);
  21.             }
  22.         }
  23.  
  24.         return "下载成功";
  25.  
  26.     }

工具静态方法类

  1. import java.io.BufferedReader;
  2. import java.io.IOException;
  3. import java.io.InputStream;
  4. import java.io.InputStreamReader;
  5. import java.net.URL;
  6. import java.net.URLConnection;
  7. import java.nio.file.Files;
  8. import java.nio.file.Path;
  9. import java.nio.file.Paths;
  10. import java.nio.file.StandardCopyOption;
  11. import java.util.ArrayList;
  12. import java.util.List;
  13. import java.util.Objects;
  14. import java.util.regex.Matcher;
  15. import java.util.regex.Pattern;
  16.  
  17. public  class StaticMethod {
  18.  
  19.     //下载总数
  20.     public static  int sumNum = 0;
  21.     //进行的页数
  22.     public static  int currPageNum = 1;
  23.     //失败次数
  24.     public static  int failTimes = 0;
  25.     public static String  download(List<String> links,String type , int pageNum){
  26.         /**
  27.          https://th.wallhaven.cc/small/4l/4ly2o2.jpg
  28.  
  29.          https://w.wallhaven.cc/full/4l/wallhaven-4ly2o2.jpg
  30.          */
  31.         for (String url : links  ) {
  32.             String realUrl = "https://w.wallhaven.cc/full/".concat(url.split("/")[4]) .concat("/wallhaven-".concat(url.split("/")[5]));
  33.             //System.out.println("当前路径: "+ realUrl);
  34.  
  35.  
  36.             String dirSec = url.split("/")[4];
  37.             String fileName = "-wallhaven-"+dirSec+"-"+url.split("/")[5];
  38.             downFiles(realUrl, type , fileName,pageNum);
  39.         }
  40.  
  41.  
  42.         return  "分任务下载完成";
  43.     }
  44.     /**
  45.      * 下载文件
  46.      * @return
  47.      */
  48.     public static   void downFiles(String url,String type,String fileName,int pageNum){
  49.  
  50.         String name = Thread.currentThread().getName();
  51.         try (InputStream ins =  getInputStream(url)) {
  52.             if(ins.available() ==0){
  53.                 return;
  54.             }
  55.             sumNum++;
  56.             String saveDir = "I:\\picture\\"+type;
  57.             Path target = Paths.get(saveDir, sumNum+fileName);
  58.             Files.createDirectories(target.getParent());
  59.             Files.copy(ins, target, StandardCopyOption.REPLACE_EXISTING);
  60.             System.out.println("第"+pageNum+"页，"+"第"+sumNum+"张图片下载完成！");
  61.         } catch (IOException e) {
  62.             System.out.println("线程"+name+"，第"+pageNum+"页，"+"下载文件时文件不存在: "+url);
  63.            // e.printStackTrace();
  64.         }
  65.     }
  66.  
  67.     public static InputStream   getInputStream(String url) throws IOException {
  68.         URL realUrl = new URL(url);
  69.  
  70.         // 打开和URL之间的连接
  71.  
  72.         URLConnection connection = realUrl.openConnection();
  73.  
  74.         //模拟浏览器通用的请求属性
  75.  
  76.         connection.setRequestProperty("accept", "*/*");
  77.  
  78.         connection.setRequestProperty("connection", "Keep-Alive");
  79.  
  80.         connection.setRequestProperty("user-agent",
  81.  
  82.                 "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
  83.  
  84.         // 建立实际的连接
  85.  
  86.         connection.connect();
  87.  
  88.         InputStream ins = connection.getInputStream();
  89.         return ins;
  90.     }
  91.  
  92.     /**
  93.      * 获取页面html,
  94.      */
  95.     public static String getHtml(String url) throws IOException {
  96.         String name = Thread.currentThread().getName();
  97.  
  98.         BufferedReader in = null;
  99.         String result="";
  100.         try (InputStream ins =  getInputStream(url)) {
  101.  
  102.             InputStreamReader read = new InputStreamReader( ins, "gb2312");// 考虑到编码格
  103.             BufferedReader bufferedReader = new BufferedReader(read);
  104.             String lineTxt = null;
  105.             //读取一行
  106.             while ((lineTxt = bufferedReader.readLine()) != null) {
  107.                 //正则表达式
  108.                 result += lineTxt;
  109.             }
  110.             read.close();
  111.             bufferedReader.close();
  112.         } catch (IOException e) {
  113.             System.out.println("线程"+name+"，页html文件不存在: " + url);
  114.            // e.printStackTrace();
  115.         }
  116.  
  117.         return result;
  118.     }
  119.  
  120.  
  121.  
  122.     /**
  123.      * 获取html所有某个相同标签上属性值
  124.      * @param html
  125.      * @param element  标签元素
  126.      * @param attr 属性
  127.      * @return
  128.      */
  129.     public static List match(String html, String element, String attr) {
  130.  
  131.         List result = new ArrayList();
  132.  
  133.         String reg = "]*?\\s" + attr + "=['\"]?(.*?)['\"]?(\\s.*?)?>";
  135.         Matcher m = Pattern.compile(reg).matcher(html);
  137.         while (m.find()) {
  139.             String r = m.group(1);
  141.             result.add(r);
  143.         }
  145.         return result;
  147.     }

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/你好赵伟/article/detail/437312
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号