热门标签
热门文章
- 1提取、拉取、推送与同步的区别（以Visual Studio与Github为例）_git同步和拉取的区别
- 2Flutter 点击输入框弹出键盘导致底部像素溢出的问题_flutter 输入法弹出时
- 3clickhouse的索引结构和查询优化_clickhouse 查询慢
- 4鸿蒙Harmony-数据持久化（Preferences）详解_鸿蒙preferences
- 5WpsOfficeExcel表格固定首行,点视图下的冻结窗格下的冻结首行
- 6ES6学习笔记6 函数的扩展_es6 函数的扩展
- 7修改edge浏览器默认搜索引擎为百度_edge的搜索引擎怎么从百度改成默认
- 8Android开发 入门篇（三） - 碎片_android onattach
- 9tiktok最火的小姐姐号推荐（抖音国际版美女网红推荐榜单）_tiktok女网红
- 10C语言编写的数码像框所有代码（jpg.c jpeg.c makefile.sh）_c 语言图片画框
当前位置: article > 正文
一个比较完善的httpWebRequest 封装,适合网络爬取及暴力破解
作者：小丑西瓜9 | 2024-03-18 01:31:01
赞
踩
httpwebrequest 封装
大家在模拟http请求的时候,对保持长连接及cookies,http头部信息等了解的不是那么深入。在各种网络请求过程中，发送N种问题。
可能问题如下：
1）登录成功后session保持
2）保证所有cookies回传到服务器
3）http头这么多，少一个，请求可能会失败
4）各种编码问题，gzip等压缩问题
为了解决这些问题,本人花了一天时间写了以下一个类,专门做http请求
- 1 using System;
- 2 using System.Collections.Generic;
- 3 using System.IO;
- 4 using System.IO.Compression;
- 5 using System.Linq;
- 6 using System.Net;
- 7 using System.Text;
- 8 using System.Threading.Tasks;
- 9
- 10 namespace ScanWeb
- 11 {
- 12 //zetee
- 13 //不能Host、Connection、User-Agent、Referer、Range、Content-Type、Content-Length、Expect、Proxy-Connection、If-Modified-Since
- 14 //等header. 这些header都是通过属性来设置的 。
- 15 public class HttpRequestClient
- 16 {
- 17 static HashSet<String> UNCHANGEHEADS = new HashSet<string>();
- 18 static HttpRequestClient()
- 19 {
- 20 UNCHANGEHEADS.Add("Host");
- 21 UNCHANGEHEADS.Add("Connection");
- 22 UNCHANGEHEADS.Add("User-Agent");
- 23 UNCHANGEHEADS.Add("Referer");
- 24 UNCHANGEHEADS.Add("Range");
- 25 UNCHANGEHEADS.Add("Content-Type");
- 26 UNCHANGEHEADS.Add("Content-Length");
- 27 UNCHANGEHEADS.Add("Expect");
- 28 UNCHANGEHEADS.Add("Proxy-Connection");
- 29 UNCHANGEHEADS.Add("If-Modified-Since");
- 30 UNCHANGEHEADS.Add("Keep-alive");
- 31 UNCHANGEHEADS.Add("Accept");
- 32
- 33 ServicePointManager.DefaultConnectionLimit = 1000;//最大连接数
- 34
- 35 }
- 36
- 37 /// <summary>
- 38 /// 默认的头
- 39 /// </summary>
- 40 public static string defaultHeaders = @"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
- 41 Accept-Encoding:gzip, deflate, sdch
- 42 Accept-Language:zh-CN,zh;q=0.8
- 43 Cache-Control:no-cache
- 44 Connection:keep-alive
- 45 Pragma:no-cache
- 46 Upgrade-Insecure-Requests:1
- 47 User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36";
- 48
- 49 /// <summary>
- 50 /// 是否跟踪cookies
- 51 /// </summary>
- 52 bool isTrackCookies = false;
- 53 /// <summary>
- 54 /// cookies 字典
- 55 /// </summary>
- 56 Dictionary<String, Cookie> cookieDic = new Dictionary<string, Cookie>();
- 57
- 58 /// <summary>
- 59 /// 平均相应时间
- 60 /// </summary>
- 61 long avgResponseMilliseconds = -1;
- 62
- 63 /// <summary>
- 64 /// 平均相应时间
- 65 /// </summary>
- 66 public long AvgResponseMilliseconds
- 67 {
- 68 get
- 69 {
- 70 return avgResponseMilliseconds;
- 71 }
- 72
- 73 set
- 74 {
- 75 if (avgResponseMilliseconds != -1)
- 76 {
- 77 avgResponseMilliseconds = value + avgResponseMilliseconds / 2;
- 78 }
- 79 else
- 80 {
- 81 avgResponseMilliseconds = value;
- 82 }
- 83
- 84 }
- 85 }
- 86
- 87 public HttpRequestClient(bool isTrackCookies = false)
- 88 {
- 89 this.isTrackCookies = isTrackCookies;
- 90 }
- 91 /// <summary>
- 92 /// http请求
- 93 /// </summary>
- 94 /// <param name="url"></param>
- 95 /// <param name="method">POST,GET</param>
- 96 /// <param name="headers">http的头部,直接拷贝谷歌请求的头部即可</param>
- 97 /// <param name="content">content,每个key,value 都要UrlEncode才行</param>
- 98 /// <param name="contentEncode">content的编码</param>
- 99 /// <param name="proxyUrl">代理url</param>
- 100 /// <returns></returns>
- 101 public string http(string url, string method, string headers, string content, Encoding contentEncode, string proxyUrl)
- 102 {
- 103 HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
- 104 request.Method = method;
- 105 if(method.Equals("GET",StringComparison.InvariantCultureIgnoreCase))
- 106 {
- 107 request.MaximumAutomaticRedirections = 100;
- 108 request.AllowAutoRedirect = false;
- 109 }
- 110
- 111 fillHeaders(request, headers);
- 112 fillProxy(request, proxyUrl);
- 113
- 114 #region 添加Post 参数
- 115 if (contentEncode == null)
- 116 {
- 117 contentEncode = Encoding.UTF8;
- 118 }
- 119 if (!string.IsNullOrWhiteSpace(content))
- 120 {
- 121 byte[] data = contentEncode.GetBytes(content);
- 122 request.ContentLength = data.Length;
- 123 using (Stream reqStream = request.GetRequestStream())
- 124 {
- 125 reqStream.Write(data, 0, data.Length);
- 126 reqStream.Close();
- 127 }
- 128 }
- 129 #endregion
- 130
- 131 HttpWebResponse response = null;
- 132 System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
- 133 try
- 134 {
- 135 sw.Start();
- 136 response = (HttpWebResponse)request.GetResponse();
- 137 sw.Stop();
- 138 AvgResponseMilliseconds = sw.ElapsedMilliseconds;
- 139 CookieCollection cc = new CookieCollection();
- 140 string cookieString = response.Headers[HttpResponseHeader.SetCookie];
- 141 if (!string.IsNullOrWhiteSpace(cookieString))
- 142 {
- 143 var spilit = cookieString.Split(';');
- 144 foreach (string item in spilit)
- 145 {
- 146 var kv = item.Split('=');
- 147 if (kv.Length == 2)
- 148 cc.Add(new Cookie(kv[0].Trim(), kv[1].Trim()));
- 149 }
- 150 }
- 151 trackCookies(cc);
- 152 }
- 153 catch (Exception ex)
- 154 {
- 155 sw.Stop();
- 156 AvgResponseMilliseconds = sw.ElapsedMilliseconds;
- 157 return "";
- 158 }
- 159
- 160 string result = getResponseBody(response);
- 161 return result;
- 162 }
- 163
- 164 /// <summary>
- 165 /// post 请求
- 166 /// </summary>
- 167 /// <param name="url"></param>
- 168 /// <param name="headers"></param>
- 169 /// <param name="content"></param>
- 170 /// <param name="contentEncode"></param>
- 171 /// <param name="proxyUrl"></param>
- 172 /// <returns></returns>
- 173 public string httpPost(string url, string headers, string content, Encoding contentEncode, string proxyUrl = null)
- 174 {
- 175 return http(url, "POST", headers, content, contentEncode, proxyUrl);
- 176 }
- 177
- 178 /// <summary>
- 179 /// get 请求
- 180 /// </summary>
- 181 /// <param name="url"></param>
- 182 /// <param name="headers"></param>
- 183 /// <param name="content"></param>
- 184 /// <param name="proxyUrl"></param>
- 185 /// <returns></returns>
- 186 public string httpGet(string url, string headers, string content=null, string proxyUrl=null)
- 187 {
- 188 return http(url, "GET", headers, null, null, proxyUrl);
- 189 }
- 190
- 191 /// <summary>
- 192 /// 填充代理
- 193 /// </summary>
- 194 /// <param name="proxyUri"></param>
- 195 private void fillProxy(HttpWebRequest request, string proxyUri)
- 196 {
- 197 if (!string.IsNullOrWhiteSpace(proxyUri))
- 198 {
- 199 WebProxy proxy = new WebProxy();
- 200 proxy.Address = new Uri(proxyUri);
- 201 request.Proxy = proxy;
- 202 }
- 203 }
- 204
- 205
- 206 /// <summary>
- 207 /// 跟踪cookies
- 208 /// </summary>
- 209 /// <param name="cookies"></param>
- 210 private void trackCookies(CookieCollection cookies)
- 211 {
- 212 if (!isTrackCookies) return;
- 213 if (cookies == null) return;
- 214 foreach (Cookie c in cookies)
- 215 {
- 216 if (cookieDic.ContainsKey(c.Name))
- 217 {
- 218 cookieDic[c.Name] = c;
- 219 }
- 220 else
- 221 {
- 222 cookieDic.Add(c.Name, c);
- 223 }
- 224 }
- 225
- 226 }
- 227
- 228 /// <summary>
- 229 /// 格式cookies
- 230 /// </summary>
- 231 /// <param name="cookies"></param>
- 232 private string getCookieStr()
- 233 {
- 234 StringBuilder sb = new StringBuilder();
- 235 foreach (KeyValuePair<string, Cookie> item in cookieDic)
- 236 {
- 237 if (!item.Value.Expired)
- 238 {
- 239 if (sb.Length == 0)
- 240 {
- 241 sb.Append(item.Key).Append("=").Append(item.Value.Value);
- 242 }
- 243 else
- 244 {
- 245 sb.Append("; ").Append(item.Key).Append(" = ").Append(item.Value.Value);
- 246 }
- 247 }
- 248 }
- 249 return sb.ToString();
- 250
- 251 }
- 252
- 253 /// <summary>
- 254 /// 填充头
- 255 /// </summary>
- 256 /// <param name="request"></param>
- 257 /// <param name="headers"></param>
- 258 private void fillHeaders(HttpWebRequest request, string headers, bool isPrint = false)
- 259 {
- 260 if (request == null) return;
- 261 if (string.IsNullOrWhiteSpace(headers)) return;
- 262 string[] hsplit = headers.Split(new String[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
- 263 foreach (string item in hsplit)
- 264 {
- 265 string[] kv = item.Split(':');
- 266 string key = kv[0].Trim();
- 267 string value = string.Join(":", kv.Skip(1)).Trim();
- 268 if (!UNCHANGEHEADS.Contains(key))
- 269 {
- 270 request.Headers.Add(key, value);
- 271 }
- 272 else
- 273 {
- 274 #region 设置http头
- 275 switch (key)
- 276 {
- 277
- 278 case "Accept":
- 279 {
- 280 request.Accept = value;
- 281 break;
- 282 }
- 283 case "Host":
- 284 {
- 285 request.Host = value;
- 286 break;
- 287 }
- 288 case "Connection":
- 289 {
- 290 if (value == "keep-alive")
- 291 {
- 292 request.KeepAlive = true;
- 293 }
- 294 else
- 295 {
- 296 request.KeepAlive = false;//just test
- 297 }
- 298 break;
- 299 }
- 300 case "Content-Type":
- 301 {
- 302 request.ContentType = value;
- 303 break;
- 304 }
- 305
- 306 case "User-Agent":
- 307 {
- 308 request.UserAgent = value;
- 309 break;
- 310 }
- 311 case "Referer":
- 312 {
- 313 request.Referer = value;
- 314 break;
- 315 }
- 316
- 317 case "Content-Length":
- 318 {
- 319 request.ContentLength = Convert.ToInt64(value);
- 320 break;
- 321 }
- 322 case "Expect":
- 323 {
- 324 request.Expect = value;
- 325 break;
- 326 }
- 327 case "If-Modified-Since":
- 328 {
- 329 request.IfModifiedSince = Convert.ToDateTime(value);
- 330 break;
- 331 }
- 332 default:
- 333 break;
- 334 }
- 335 #endregion
- 336 }
- 337 }
- 338 CookieCollection cc = new CookieCollection();
- 339 string cookieString = request.Headers[HttpRequestHeader.Cookie];
- 340 if (!string.IsNullOrWhiteSpace(cookieString))
- 341 {
- 342 var spilit = cookieString.Split(';');
- 343 foreach (string item in spilit)
- 344 {
- 345 var kv = item.Split('=');
- 346 if (kv.Length == 2)
- 347 cc.Add(new Cookie(kv[0].Trim(), kv[1].Trim()));
- 348 }
- 349 }
- 350 trackCookies(cc);
- 351 if (!isTrackCookies)
- 352 {
- 353 request.Headers[HttpRequestHeader.Cookie] = "";
- 354 }
- 355 else
- 356 {
- 357 request.Headers[HttpRequestHeader.Cookie] = getCookieStr();
- 358 }
- 359
- 360 #region 打印头
- 361 if (isPrint)
- 362 {
- 363 for (int i = 0; i < request.Headers.AllKeys.Length; i++)
- 364 {
- 365 string key = request.Headers.AllKeys[i];
- 366 System.Console.WriteLine(key + ":" + request.Headers[key]);
- 367 }
- 368 }
- 369 #endregion
- 370
- 371 }
- 372
- 373
- 374 /// <summary>
- 375 /// 打印ResponseHeaders
- 376 /// </summary>
- 377 /// <param name="response"></param>
- 378 private void printResponseHeaders(HttpWebResponse response)
- 379 {
- 380 #region 打印头
- 381 if (response == null) return;
- 382 for (int i = 0; i < response.Headers.AllKeys.Length; i++)
- 383 {
- 384 string key = response.Headers.AllKeys[i];
- 385 System.Console.WriteLine(key + ":" + response.Headers[key]);
- 386 }
- 387 #endregion
- 388 }
- 389
- 390
- 391 /// <summary>
- 392 /// 返回body内容
- 393 /// </summary>
- 394 /// <param name="response"></param>
- 395 /// <returns></returns>
- 396 private string getResponseBody(HttpWebResponse response)
- 397 {
- 398 Encoding defaultEncode = Encoding.UTF8;
- 399 string contentType = response.ContentType;
- 400 if (contentType != null)
- 401 {
- 402 if (contentType.ToLower().Contains("gb2312"))
- 403 {
- 404 defaultEncode = Encoding.GetEncoding("gb2312");
- 405 }
- 406 else if (contentType.ToLower().Contains("gbk"))
- 407 {
- 408 defaultEncode = Encoding.GetEncoding("gbk");
- 409 }
- 410 else if (contentType.ToLower().Contains("zh-cn"))
- 411 {
- 412 defaultEncode = Encoding.GetEncoding("zh-cn");
- 413 }
- 414 }
- 415
- 416 string responseBody = string.Empty;
- 417 if (response.ContentEncoding.ToLower().Contains("gzip"))
- 418 {
- 419 using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
- 420 {
- 421 using (StreamReader reader = new StreamReader(stream))
- 422 {
- 423 responseBody = reader.ReadToEnd();
- 424 }
- 425 }
- 426 }
- 427 else if (response.ContentEncoding.ToLower().Contains("deflate"))
- 428 {
- 429 using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
- 430 {
- 431 using (StreamReader reader = new StreamReader(stream, defaultEncode))
- 432 {
- 433 responseBody = reader.ReadToEnd();
- 434 }
- 435 }
- 436 }
- 437 else
- 438 {
- 439 using (Stream stream = response.GetResponseStream())
- 440 {
- 441 using (StreamReader reader = new StreamReader(stream, defaultEncode))
- 442 {
- 443 responseBody = reader.ReadToEnd();
- 444 }
- 445 }
- 446 }
- 447 return responseBody;
- 448 }
- 449
- 450
- 451 public static string UrlEncode(string item, Encoding code)
- 452 {
- 453 return System.Web.HttpUtility.UrlEncode(item.Trim('\t').Trim(), Encoding.GetEncoding("gb2312"));
- 454 }
- 455
- 456 public static string UrlEncodeByGB2312(string item)
- 457 {
- 458 return UrlEncode(item, Encoding.GetEncoding("gb2312"));
- 459 }
- 460
- 461
- 462 public static string UrlEncodeByUTF8(string item)
- 463 {
- 464 return UrlEncode(item, Encoding.GetEncoding("utf-8"));
- 465 }
- 466
- 467 public static string HtmlDecode(string item)
- 468 {
- 469 return WebUtility.HtmlDecode(item.Trim('\t').Trim());
- 470 }
- 471
- 472 }
- 473 }
使用方式:
1)打开谷歌浏览器,或者F12
复制Request Headers 里面的所有内容,然后执行代码:
就这样,你会惊喜的发现,卧槽!反回来的值和谷歌上显示值一个样子,
只要域名没变化,HttpRequestClient 对象就不要去改变, 多线程请使用ThreadLocal<HttpRequestClient >
配合我很久之前写的多线类 QueueThreadBase 让你起飞.
你想暴力破解网站登录密码吗?基本思路如下:
1)强大的用户名+密码字典
2)多线程Http+代理(代理可以不用,如果服务器做了ip限制,那么代理就非常有用了,最好是透明的http代理,并且有规则剔除慢的代理)
3)验证码破解.(只要验证码不复杂,在某宝就能买的dll 可用,1000块钱上下)
声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/小丑西瓜9/article/detail/259396
推荐阅读
相关标签
