C# 模拟浏览器发起http Get请求,Post请求,支持GZIP响应体

后端C#/ASP.NET / C#公用方法/函数/组件
字数统计: 1.3k   |   阅读时长≈ 6 分钟

@[TOC](C# 模拟浏览器发起http Get请求,Post请求,支持GZIP响应体)

场景说明

当需要调用接口、下载其它网页数据，可以使用第三方组件（例如RestSharp），但我只想简单用一下，以下主要是自行封装的示例代码。容易漏的是对C# 非托管代码的资源的释放（例如：StreamReader.Close、HttpWebRequest.Abort、Stream.Dispose），并且支持GZIP响应体解析。网络上有很多种实现，但我依然要分享我用了几年的封装，因为多数代码没有注意非托管代码的资源的释放，这样的代码运行久了容易内存泄漏。另外就是对Url的编码需要自行按参数去处理，很麻烦，干脆也封装一下。

代码如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;

public class ToolX
{
    /// <summary>
    /// 发起Get请求，并接收响应体为字符串
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="encode">编码格式</param>
    /// <param name="dicHeader">请求头</param>
    /// <returns></returns>
    public string HttpGet(string url, Encoding encode = null, Dictionary<string, string> dicHeader = null) 
    {
        string retStr = string.Empty;
        encode = encode ?? Encoding.UTF8;
        try
        {
            HttpWebRequest hwr = (HttpWebRequest)WebRequest.Create(UrlEncode(url, encode));
            hwr.Method = "GET";
            hwr.ContentType = "text/html; charset=utf-8";
            hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36";
            hwr.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            hwr.KeepAlive = false;
            hwr.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (dicHeader != null)
            {
                foreach (var kv in dicHeader)
                {
                    switch (kv.Key)
                    {
                        //部分请求头不能像这样hwr.Headers[kv.Key] = kv.Value，需要的自行写
                        case "ContentType": hwr.ContentType = kv.Value; break;
                        case "Accept": hwr.Accept = kv.Value; break;
                        case "UserAgent": hwr.UserAgent = kv.Value; break;
                        case "Referer": hwr.Referer = UrlEncode(kv.Value, encode); break;
                        default: hwr.Headers[kv.Key] = kv.Value; break;
                    }
                }
            }

            //下载页面
            HttpWebResponse httpresponse = (HttpWebResponse)hwr.GetResponse();
            retStr = StreamToString(httpresponse.GetResponseStream(), encode, httpresponse.ContentEncoding.ToLower().Trim() == "gzip");
            httpresponse.Close();
            hwr.Abort();//重要，不然会占用TCP链接资源
        }
        catch (WebException ex)
        {
            //日志自行实现
            //ex.Status;  
            var response = ex.Response as System.Net.HttpWebResponse;
            if (response != null)
            {
                //Console.WriteLine("HTTP Status Code: " + (int)response.StatusCode);
                //string errResopnse = StreamToString(response.GetResponseStream(), encode, response.ContentEncoding.ToLower().Trim() == "gzip");
            }
        }
        catch (Exception ex)
        {
            //日志自行实现
        }
        finally
        {
            GC.Collect();
        }

        return retStr;
    }

    /// <summary>
    /// 发起Post请求，并接收响应体为字符串
    /// </summary>
    /// <param name="url">网址</param>
    /// <param name="postBody">请求体，目前只能传字符串、byte[]、Stream</param>
    /// <param name="encode">编码格式</param>
    /// <param name="dicHeader">请求头</param>
    /// <returns></returns>
    public string HttpPost(string url, object postBody, Encoding encode = null, Dictionary<string, string> dicHeader = null)
    {
        string retStr = string.Empty;
        encode = encode ?? Encoding.UTF8;
        try
        {
            HttpWebRequest hwr = (HttpWebRequest)WebRequest.Create(UrlEncode(url, encode));
            hwr.Method = "POST";
            hwr.ContentType = "application/x-www-form-urlencoded";
            hwr.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36";
            hwr.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            hwr.KeepAlive = false;
            hwr.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (dicHeader != null)
            {
                foreach (var kv in dicHeader)
                {
                    switch (kv.Key)
                    {
                        //部分请求头不能像这样hwr.Headers[kv.Key] = kv.Value，需要的自行写
                        case "ContentType": hwr.ContentType = kv.Value; break;
                        case "Accept": hwr.Accept = kv.Value; break;
                        case "UserAgent": hwr.UserAgent = kv.Value; break;
                        case "Referer": hwr.Referer = UrlEncode(kv.Value, encode); break;
                        default: hwr.Headers[kv.Key] = kv.Value; break;
                    }
                }
            }

            Stream postDataStream = hwr.GetRequestStream();
            //处理请求体为byte[]
            byte[] postBytes = null;
            if (postBody != null)
            {
                var tp = postBody.GetType();
                if (tp == typeof(string))
                {
                    encode.GetBytes(postBody + "");
                }
                else if(tp == typeof(byte[]))
                {
                    postBytes = postBody as byte[];
                }
                else if(postBody is Stream)
                {
                    using(Stream sm = postBody as Stream)
                    {
                        postBytes = new byte[sm.Length];
                        sm.Position = 0;
                        sm.Read(postBytes, 0, Convert.ToInt32(sm.Length));
                    }
                }
                else
                {
                    throw new Exception("不支持的请求体类型");
                }
            }
                
            //发送请求体
            if (postBytes != null)
            {
                hwr.ContentLength = postBytes.Length;
                postDataStream.Write(postBytes, 0, postBytes.Length);
            }

            //下载页面
            HttpWebResponse httpresponse = (HttpWebResponse)hwr.GetResponse();
            retStr = StreamToString(httpresponse.GetResponseStream(), encode, httpresponse.ContentEncoding.ToLower().Trim() == "gzip");
            httpresponse.Close();
            hwr.Abort();//重要，不然会占用TCP链接资源
        }
        catch (WebException ex)
        {
            //日志自行实现
            //ex.Status;  
            var response = ex.Response as System.Net.HttpWebResponse;
            if (response != null)
            {
                //Console.WriteLine("HTTP Status Code: " + (int)response.StatusCode);
                //string errResopnse = StreamToString(response.GetResponseStream(), encode, response.ContentEncoding.ToLower().Trim() == "gzip");
            }
        }
        catch (Exception ex)
        {
            //日志自行实现
        }
        finally
        {
            GC.Collect();
        }

        return retStr;
    }

    /// <summary>
    /// 读取流为字符串
    /// </summary>
    /// <param name="ream">注意，该流会被释放</param>
    /// <param name="encode"></param>
    /// <param name="isGzip"></param>
    /// <returns></returns>
    public string StreamToString(Stream ream, Encoding encode, bool isGzip = false)
    {
        string retStr = null;
        if (ream != null)
        {
            List<byte> body = new List<byte>();
            int b = -1;
            byte[] bts = new byte[102400];
            while ((b = ream.Read(bts, 0, bts.Length)) > 0) body.AddRange(bts.Take(b));
            ream.Close(); ream.Dispose();
            ream = new MemoryStream(body.ToArray());
            if (isGzip)
            {
                ream = new GZipStream(ream, CompressionMode.Decompress);
            }
            StreamReader sr = new StreamReader(ream, encode);
            retStr = sr.ReadToEnd();
            sr.Close(); sr.Dispose(); ream.Close(); ream.Close(); 
        }
        return retStr;
    }


    /// <summary>
    /// 仅对字符大于255的进行编码
    /// </summary>
    /// <param name="url"></param>
    /// <param name="encode"></param>
    /// <returns></returns>
    public string UrlEncode(string url, Encoding encode = null)
    {
        if (string.IsNullOrWhiteSpace(url)) return url;
        encode = encode ?? Encoding.UTF8;
        StringBuilder sb = new StringBuilder();
        foreach (var c in url)
        {
            if (c <= 255) sb.Append(c);
            else sb.Append(System.Web.HttpUtility.UrlEncode("" + c, encode));
        }
        return sb.ToString();
    }
}

调用示例

1
2
string ss = new ToolX().HttpGet("https://www.80juchang.com/");
string ss2 = new ToolX().HttpPost("https://www.80juchang.com/", "a=中文");

原文链接：http://blog.albsz.cn/2020-07-17-csharpHttp.html

打赏
  • 版权声明： 本博客所有文章除特别声明外，著作权归作者所有。转载请注明出处！

撸代码不易，给点物质上的支持吧～

支付宝
微信