關(guān)于HTTP協(xié)議
http協(xié)議是一種無(wú)狀態(tài)協(xié)議。這是首先要明確的，客戶端（client）發(fā)送一個(gè)請(qǐng)求，服務(wù)端（server）收到之后，根據(jù)請(qǐng)求的URL和HTTP頭信息，給出相應(yīng)的答案。505,404,400等，一般正確的200，服務(wù)器除了IP和UserAgent等常用信息之外，服務(wù)器無(wú)法知道具體的標(biāo)示，也就是說(shuō)服務(wù)器無(wú)法知道這個(gè)請(qǐng)求來(lái)自哪個(gè)客戶端，OK！
那么就引入了Cookie的概念，服務(wù)器一般用cookie去標(biāo)示客戶端，可見(jiàn)cookie對(duì)于現(xiàn)在web系統(tǒng)的重要性，如果沒(méi)有cookie現(xiàn)在的web啥不是。
也就是說(shuō)Cookie的web交互核心之一

要實(shí)現(xiàn)web自動(dòng)化操作，就要控制Cookie以及http頭等信息，現(xiàn)在假設(shè)一個(gè)場(chǎng)景，
QQ郵箱登陸：
1，登陸QQ郵箱的主地址（http://mail.qq.com）
?
請(qǐng)求頭如上
?

響應(yīng)內(nèi)容，跳轉(zhuǎn)到登陸頁(yè)（因?yàn)闆](méi)有登陸后的cookie標(biāo)示）
2，會(huì)經(jīng)過(guò)幾個(gè)跳轉(zhuǎn)步驟之后跳轉(zhuǎn)到HTTPS登陸（https://mail.qq.com/cgi-bin/loginpage?&res=local）
?

3，輸入賬號(hào)登陸
輸入密碼后會(huì)跳轉(zhuǎn)到 使用get方式提交表單，如果登陸成功會(huì)寫Cookie
?

4，登陸成功之后我們?cè)俅芜M(jìn)入通過(guò)mail.qq.com域名進(jìn)入，也會(huì)跳轉(zhuǎn)到登陸頁(yè)，但是由于請(qǐng)求頭中的cookie已經(jīng)包含登陸標(biāo)示，所以會(huì)直接跳轉(zhuǎn)到郵箱url
?

重上述過(guò)程可以看出用戶驗(yàn)證和識(shí)別都是依賴于Cookie，那么我們就需要一個(gè)能夠控制Cookie，能夠定制HTTP頭，能發(fā)起HTTP請(qǐng)求的功能模塊，也許有人會(huì)問(wèn)為什么不用webClient或webBrowser，這個(gè)要重2方面說(shuō)明，一個(gè)是性能webBrowser會(huì)加載頁(yè)面所有的東西，而且共享IE瀏覽器的信息，我們是需要獨(dú)立的，因?yàn)橛锌赡軙?huì)一個(gè)程序同時(shí)登陸多個(gè)用戶，WebClient在提交Post數(shù)據(jù)一個(gè)獲取HTTP頭方面比較弱，所有就只能是webReques和自定義Cookie組合

Cookie自定義管理實(shí)現(xiàn)

public class CookieManager { /** * Parse a Cookie: header into individual tokens according to RFC 2109. */ private class CookieTokenizer { /** * Upper bound on the number of cookie tokens to accept. The limit is * based on the 4 different attributes (4.3.4) and the 20 cookie minimum * (6.3) given in RFC 2109 multiplied by 2 to accomodate the 2 tokens in * each name=value pair ("JSESSIONID=1234" is 2 tokens). */ private const int MAX_COOKIE_TOKENS = 4 * 20 * 2;? /** * Array of cookie tokens. Even indices contain name tokens while odd * indices contain value tokens (or null). */ public string[] tokens = new string[MAX_COOKIE_TOKENS];? /** * Number of cookie tokens currently in the tokens[] array. */ private int numTokens = 0;? /** * Parse a name=value pair from the Cookie: header. * * @param cookies The Cookie: header to parse * @param beginIndex The index in cookies to begin parsing from, inclusive */ private int parseNameValue(string cookies, int beginIndex) { int length = cookies.Length; int index = beginIndex;? while (index < length) { switch (cookies[index]) { case ';': case ',': // Found end of name token without value tokens[numTokens] = cookies.Substring(beginIndex, index - beginIndex).Trim(); if (tokens[numTokens].Length > 0) { numTokens++; tokens[numTokens] = null; numTokens++; } return index + 1;? case '=': // Found end of name token with value tokens[numTokens] = cookies.Substring(beginIndex, index - beginIndex).Trim(); numTokens++; return parseValue(cookies, index + 1);? case '"': // Skip past quoted span do index++; while (cookies[index] != '"'); break; }? index++; }? if (index > beginIndex) { // Found end of name token without value tokens[numTokens] = cookies.Substring(beginIndex, index - beginIndex).Trim(); if (tokens[numTokens].Length > 0) { numTokens++; tokens[numTokens] = null; numTokens++; } }? return index; }? /** * Parse the name=value tokens from a Cookie: header. * * @param cookies The Cookie: header to parse */ public int tokenize(string cookies) { numTokens = 0;? if (cookies != null) { try { // Advance through cookies, parsing name=value pairs int length = cookies.Length; int index = 0; while (index < length) index = parseNameValue(cookies, index); } catch (Exception e) { // Filled up the tokens[] array } //catch (IndexOutOfBoundsException e) //{ // // Walked off the end of the cookies header //} }? return numTokens; }? /** * Return the number of cookie tokens parsed from the Cookie: header. */ public int getNumTokens() { return numTokens; }? /** * Returns a given cookie token from the Cookie: header. * * @param index The index of the cookie token to return */ public String tokenAt(int index) { return tokens[index]; }? /** * Parse the value token from a name=value pair. * * @param cookies The Cookie: header to parse * @param beginIndex The index in cookies to begin parsing from, inclusive */ private int parseValue(String cookies, int beginIndex) { int length = cookies.Length; int index = beginIndex;? while (index < length) { switch (cookies[index]) { case ';': case ',': // Found end of value token tokens[numTokens] = cookies.Substring(beginIndex, index - beginIndex).Trim(); numTokens++; return index + 1;? case '"': // Skip past quoted span do index++; while (cookies[index] != '"'); break; }? index++; }? // Found end of value token tokens[numTokens] = cookies.Substring(beginIndex, index - beginIndex).Trim(); numTokens++;? return index; } }? static Regex regExpires = new Regex(@"expires\=[\s]*?[\w]+.+?(?=(,|;|[\w-]+\=|$))", RegexOptions.IgnoreCase); static Regex regHttpOnly = new Regex(@"httponly([\s]*?|,)", RegexOptions.IgnoreCase); static CookieTokenizer ct = new CookieTokenizer(); static string[] systemKey = new string[] { "expires","domain","path","max-age","version" }; static public List<Cookie> ParseCookieHeader(string cookieHeader, string defaultDomain) { List<Cookie> cList = new List<Cookie>(); var masExpires = regExpires.Matches(cookieHeader); foreach (Match maExpires in masExpires) { string dateText = maExpires.Value.Trim().Substring(8, maExpires.Value.Trim().Length - 8); if (dateText.IndexOf(",") == dateText.Length - 1 || dateText.IndexOf(";") == dateText.Length - 1) { dateText = dateText.Substring(0, dateText.Length - 1); } if (dateText.IndexOf(",") == 3) { dateText = dateText.Substring(3, dateText.Length - 3); } DateTime date = DateTime.Parse(dateText); cookieHeader = cookieHeader.Replace(maExpires.Value, "expires=" + date.Ticks.ToString() + ";"); } cookieHeader = regHttpOnly.Replace(cookieHeader, ""); int count = ct.tokenize(cookieHeader); string key = ""; Cookie cookie = null; for (int i = 0; i < count; i++) { if (i % 2 == 0) { key = ct.tokens[i]; } else { if (key != "") { if (!systemKey.Contains(key.ToLower())) { cookie = new Cookie(); cookie.Name = key; cookie.Value = ct.tokens[i]; cookie.Path = "/"; cookie.Expires = DateTime.Now.AddDays(1); cookie.Domain = defaultDomain; if (cList.Count(p => p.Name.ToLower().Trim() == key.ToLower().Trim()) > 0) { cList.Remove(cList.Where(p => p.Name.ToLower().Trim() == key.ToLower().Trim()).Take(1).Single()); } cList.Add(cookie); } else { if (cookie != null) { if (key.ToLower() == systemKey[0]) { try { cookie.Expires = cookie.Expires.AddMilliseconds(double.Parse(ct.tokens[i]) / 10000); } catch { cookie.Expires = DateTime.Now.AddDays(1); } } else if (key.ToLower() == systemKey[1]) { cookie.Domain = ct.tokens[i]; } else if (key.ToLower() == systemKey[2]) { cookie.Path = ct.tokens[i]; } else if (key.ToLower() == systemKey[3]) { try { cookie.Expires = cookie.Expires.AddSeconds(double.Parse(ct.tokens[i])); } catch { cookie.Expires = DateTime.Now.AddDays(1); } } } }? } } } return cList; } List<Cookie> cookieList = new List<Cookie>(); public void SetCookie(CookieCollection cookies) { foreach (Cookie cookie in cookies) { if (cookieList.Count(p => p.Name == cookie.Name && p.Domain.ToLower() == cookie.Domain.ToLower()) > 0) { var tc = cookieList.Where(p => p.Name == cookie.Name && p.Domain.ToLower() == cookie.Domain.ToLower()).Take(1).Single(); cookieList.Remove(tc); } cookieList.Add(cookie); } } public void SetCookie(List<Cookie> cookies) { CookieCollection cc = new CookieCollection(); foreach (Cookie cookie in cookies) { cc.Add(cookie); } SetCookie(cc); } public void SetCookie(string cookieHeader, string defaultDomain) { SetCookie(ParseCookieHeader(cookieHeader, defaultDomain)); } public void SetCookie(Cookie ck) { CookieCollection cc = new CookieCollection(); cc.Add(ck); SetCookie(cc); } public string GetCookieHeader(string host) { var whe = GetCookies(host); return GetCookieString(whe); }? static Regex regDomain = new Regex(@"[\w]+\.(org\.cn|net\.cn|com\.cn|com|net|org|gov|cc|biz|info|cn|hk)+$"); public CookieCollection GetCookies(string serverHost) { List<string> domainList = new List<string>(); string domain = regDomain.Match(serverHost).Value; string host = serverHost.ToLower().Replace("www.", ""); host = host.Replace(domain, "domain"); string[] pars = host.Split('.'); if (pars.Length > 1) {? string tmp = ""; for (int i = pars.Length - 1; i > -1; i--) { if (pars[i] == "domain") continue; tmp = pars[i] + "." + tmp; domainList.Add(tmp + domain); domainList.Add("." + tmp + domain); } } domainList.Add(serverHost); domainList.Add(domain); domainList.Add("." + domain); CookieCollection cc = new CookieCollection(); var whe = cookieList.Where(p => domainList.Contains(p.Domain.ToLower())); foreach (var cookie in whe) { cc.Add(cookie); } return cc; }? public CookieCollection Convert(List<Cookie> cks) { CookieCollection cc = new CookieCollection(); foreach (Cookie item in cks) { cc.Add(item); } return cc; }? public List<Cookie> Convert(CookieCollection cks) { List<Cookie> cc = new List<Cookie>(); foreach (Cookie item in cks) { cc.Add(item); } return cc; }? private string GetCookieString(CookieCollection cks) { StringBuilder strCookie = new StringBuilder(); foreach (Cookie cookie in cks) { strCookie.Append(cookie.Name); strCookie.Append("="); strCookie.Append(cookie.Value); strCookie.Append("; "); } if (strCookie.Length > 3) strCookie = strCookie.Remove(strCookie.Length - 2, 2); return strCookie.ToString(); }? public void SetAllCookieToDomain(string domain) { var atCookie = Convert(GetCookies(domain)); var needCookie = cookieList.Where(p => !atCookie.Contains(p)).ToArray(); for (int i = 0; i < needCookie.Length; i++) { Cookie item = needCookie[i]; cookieList.Add(new Cookie() { Domain = domain, Expired = item.Expired, Expires = item.Expires, Name = item.Name, Path = item.Path, Value = item.Value }); }? }? public void Clear() { cookieList.Clear(); }? public void RemoveCookie(string name, string doamin = null) { var cks = new List<Cookie>(); var temp = new Cookie[cookieList.Count]; cookieList.CopyTo(temp); cks = temp.ToList(); if (!string.IsNullOrEmpty(doamin)) { cks = Convert(GetCookies(doamin)); } foreach (Cookie cookie in cks) { if (cookie.Name.Trim() == name.Trim()) { cookieList.Remove(cookie); } }? }? public void SetIECookie(string host = null) { var cks = cookieList; if (!string.IsNullOrEmpty(host)) { cks = Convert(GetCookies(host)); } foreach (var cookie in cookieList) { for (int i = 0; i < 5 && !WinAPI.InternetSetCookie("http://" + (cookie.Domain.StartsWith(".") ? "www" : "") + cookie.Domain, cookie.Name, cookie.Value + ";expires=" + cookie.Expires.ToGMTString()); i++) ;? } } } 自定義web操作類
abstract public class WebAction { /* 類名:web操作基礎(chǔ)支持類 * 描述:提供web操作基礎(chǔ)接口 * 創(chuàng)建日期:2011-10-25 * 版本:0.4 * 作者:by rolends986 */ /* * 版本更新記錄 * 0.1 基本代碼的構(gòu)建與調(diào)試 * 0.2 修改主入口方法,實(shí)現(xiàn)多參數(shù)化定義 2011-11-1 * 0.3 添加SetUseUnsafeHeaderParsing功能,修訂lock邏輯,刪除url編碼邏輯(會(huì)導(dǎo)致部分服務(wù)器header解析問(wèn)題) 2011-12-2 * 0.4 新增代理控制邏輯,修改useUnsafeHeaderParsing參數(shù),添加資源釋放邏輯 2011-12-12 */ static WebAction() { DefaultConnectionLimit = 1000; KeepAliveTime = 10 * 1000; KeepAliveInterval = 300; }? protected CookieManager _cookieManager = new CookieManager(); protected XQSoft.Common.Log LogObject { get { return LogManager.Logs[LogName]; } } string _logName = ""; virtual protected string LogName { get { return _logName; } } public WebAction() {? } public WebAction(string logName) { _logName = logName; } public const string _userAgent_FF = " Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20100101 Firefox/11.0"; public const string _userAgent_IE = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C; .NET4.0E)"; public const string _defaultAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; public const string _defaultAcceptLanguage = "zh-cn,zh;q=0.5"; public const string _defaultAcceptCharset = "GB2312,utf-8;q=0.7,*;q=0.7"; public static int DefaultConnectionLimit { get; set; } static public int KeepAliveTime { get; set; } static public int KeepAliveInterval { get; set; } public bool EnableProxy { get; set; } ProxyInfo _proxyInfo = null; public ProxyInfo ProxyInfo { get { return _proxyInfo; } protected set { _proxyInfo = value; } } public string Key { get { return new Guid().ToString(); } } static object sslLock = new object(); static public bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) { //if (sender is HttpWebRequest) //{ // var request = sender as HttpWebRequest; // if (request.ClientCertificates.Count > 0) // return false; //} return true; // Always accept }?? virtual protected void ChangeProxy() { _proxyInfo = ProxyManager.GetAvailableProxy(this.Key); }? /* 新方法 by rolends1986 * 2011-10-27 1,支持自定義Content-Type 2,封裝編碼,自身維護(hù)表單鍵值轉(zhuǎn)換 3,支持各種參數(shù)的自定義 4,實(shí)現(xiàn)自動(dòng)編碼 5,實(shí)現(xiàn)CA文件指定 */ virtual protected T GetWebData<T>(string url, string charset = null, string referer = null, PostInfo postInfo = null, bool? useProxy = null, NameValueCollection headers = null, string userAgent = null, Certificates certificates = null, Version protocol = null, bool? allowAutoRedirect = false, bool? keepAlive = null, string accept = null, string acceptLanguage = null, string acceptCharset = null, string urlEncoding = null, RequestCachePolicy cachePolicy = null) { System.Net.ServicePointManager.DefaultConnectionLimit = DefaultConnectionLimit; //System.Net.ServicePointManager.SetTcpKeepAlive(true, KeepAliveTime, KeepAliveInterval); //SetUseUnsafeHeaderParsing(useUnsafeHeaderParsing); var uri = new Uri(url); //url = EncodeUrl(url, urlEncoding); var request = (HttpWebRequest)WebRequest.Create(url); request.ServicePoint.Expect100Continue = false; request.Proxy = null; if (useProxy.HasValue) { if (useProxy.Value) { SetProxy(request); }? } else { if (EnableProxy) { SetProxy(request); } }? #region set default request.KeepAlive = false; request.AllowAutoRedirect = false; request.UserAgent = _userAgent_FF; request.Accept = _defaultAccept; request.Headers.Add(HttpRequestHeader.AcceptLanguage, _defaultAcceptLanguage); request.Headers.Add(HttpRequestHeader.AcceptCharset, _defaultAcceptCharset); request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore); request.Method = "get"; #endregion if (url.ToLower().IndexOf("https") == 0) { if (certificates != null) { X509CertificateCollection crts = null; if (certificates.IsAuto) { crts = GetCertificates(uri); } else { crts = certificates.CertificateCollection; } if (crts == null) ThrowException(505, url); request.ClientCertificates = crts; } request.ProtocolVersion = HttpVersion.Version10; } //request.Host = uri.Host; if (allowAutoRedirect.HasValue) request.AllowAutoRedirect = allowAutoRedirect.Value; //if (keepAlive.HasValue) request.KeepAlive = keepAlive.Value;//由于手動(dòng)釋放了資源,keepalive設(shè)置不再有效 if (!string.IsNullOrEmpty(userAgent)) request.UserAgent = userAgent; if (!string.IsNullOrEmpty(accept)) request.Accept = accept; if (!string.IsNullOrEmpty(acceptLanguage)) { if (request.Headers[HttpRequestHeader.AcceptLanguage] == null) request.Headers.Add(HttpRequestHeader.AcceptLanguage, acceptLanguage); else request.Headers[HttpRequestHeader.AcceptLanguage] = acceptLanguage; } if (!string.IsNullOrEmpty(acceptCharset)) { if (request.Headers[HttpRequestHeader.AcceptCharset] == null) request.Headers.Add(HttpRequestHeader.AcceptCharset, acceptCharset); else request.Headers[HttpRequestHeader.AcceptCharset] = acceptCharset; } if (!string.IsNullOrEmpty(referer)) request.Referer = referer; if (cachePolicy != null) request.CachePolicy = cachePolicy; if (protocol != null) request.ProtocolVersion = protocol; try { if (headers != null) foreach (var nv in headers.AllKeys) { request.Headers.Add(nv, headers[nv]); } } catch (Exception ex) { DisposeRequest(request); //request header error 502 ThrowException(502, ex.Message); } string requestCookie = _cookieManager.GetCookieHeader(uri.Host); if (!String.IsNullOrEmpty(requestCookie)) { request.Headers.Add(HttpRequestHeader.Cookie, requestCookie); } if (postInfo != null) { request.Method = "post"; byte[] byteArray = postInfo.GetPostData(); request.ContentType = postInfo.GetContentType(); request.ContentLength = byteArray.Length; Stream dataStream = request.GetRequestStream(); dataStream.Write(byteArray, 0, byteArray.Length); dataStream.Close(); } WebResponse response = null; try { if (url.ToLower().IndexOf("https") == 0) { lock (sslLock) { if (certificates != null) { if (ServicePointManager.ServerCertificateValidationCallback != null) ServicePointManager.ServerCertificateValidationCallback -= CheckValidationResult; } else { if (ServicePointManager.ServerCertificateValidationCallback == null) ServicePointManager.ServerCertificateValidationCallback += CheckValidationResult; } response = request.GetResponse(); } } else { response = request.GetResponse(); } } catch (Exception ex) { DisposeRequest(request); DisposeResponse(response); //get response error 503 ThrowException(503, ex.Message); } string cookie = response.Headers.Get("Set-Cookie"); if (!String.IsNullOrEmpty(cookie)) { _cookieManager.SetCookie(cookie, uri.Host); } var sm = response.GetResponseStream(); if (typeof(T) == typeof(string)) { if (!String.IsNullOrEmpty(response.Headers["Content-Type"])) { string[] ct = response.Headers["Content-Type"].Split(';'); if (ct.Length > 1) { charset = ct[1].Split('=')[1];//set server response encoding } } string html = GetHtml(sm, charset); T result = (T)(object)html; DisposeRequest(request); DisposeResponse(response); return result; } else if (typeof(Image) == typeof(T)) { try { Image original = Image.FromStream(sm); T result = (T)(object)original; DisposeRequest(request); DisposeResponse(response); return result; } catch (Exception ex) { //to image error 504 DisposeRequest(request); DisposeResponse(response); ThrowException(504, ex.Message); return default(T); }? } else if (typeof(ResponseLocation) == typeof(T)) { ResponseLocation rl = new ResponseLocation() { Html = GetHtml(sm, charset), Url = response.Headers["Location"] }; T result = (T)(object)rl; DisposeRequest(request); DisposeResponse(response); return result;? } else { T result = (T)(object)GetData(sm); DisposeRequest(request); DisposeResponse(response); return result; } }? private void DisposeResponse(WebResponse response) { try { response.GetResponseStream().Close(); } catch { } try { response.Close(); } catch { } try { response = null; } catch { } }? private void DisposeRequest(HttpWebRequest request) { try { try { request.GetRequestStream().Close(); } catch { } try { request.Abort(); } catch { } try { request = null; } catch { }??? } catch { } }? private void SetProxy(HttpWebRequest request) { if (ProxyInfo == null) ThrowException(533, "代理實(shí)例為空,請(qǐng)先實(shí)例化"); request.Proxy = new WebProxy(ProxyInfo.IPAddress.ToString(), ProxyInfo.Port); }? public static bool SetUseUnsafeHeaderParsing(bool boolVal) { try { Assembly assem = Assembly.GetAssembly(typeof(System.Net.Configuration.SettingsSection)); if (assem == null) return false; Type assemType = assem.GetType("System.Net.Configuration.SettingsSectionInternal"); if (assemType == null) return false; object obj = assemType.InvokeMember("Section", BindingFlags.Static | BindingFlags.GetProperty | BindingFlags.NonPublic, null, null, new object[] { }); if (obj == null) return false; FieldInfo fieldInfo = assemType.GetField("useUnsafeHeaderParsing", BindingFlags.NonPublic | BindingFlags.Instance); if (fieldInfo == null) return false; fieldInfo.SetValue(obj, boolVal); } catch { } return true; }? private string EncodeUrl(string url, string code) { if (string.IsNullOrEmpty(code)) return url; Encoding urlCode = Encoding.ASCII; if (!String.IsNullOrEmpty(code)) { urlCode = Encoding.GetEncoding(code); } int pIndex = url.IndexOf('?'); if (url.Length - 1 <= pIndex) return url; if (pIndex > 1) { string[] its = url.Substring(pIndex + 1).Split('&'); StringBuilder np = new StringBuilder(); foreach (var nv in its) { string name = ""; string value = ""; int cIndex = nv.IndexOf("="); if (cIndex < 0) name = nv; else { name = nv.Substring(0, cIndex); if (nv.Length - 1 > cIndex) value = nv.Substring(cIndex + 1); } np.Append(UrlUnit.UrlEncode(name, urlCode)); np.Append("="); np.Append(UrlUnit.UrlEncode(value, urlCode)); np.Append("&"); } url = url.Substring(0, pIndex + 1) + np.Remove(np.Length - 1, 1).ToString(); } return url; }? public byte[] GZipDecompress(byte[] gzip) { using (GZipStream stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress)) { const int size = 4096; byte[] buffer = new byte[size]; using (MemoryStream memory = new MemoryStream()) { int count = 0; do { count = stream.Read(buffer, 0, size); if (count > 0) { memory.Write(buffer, 0, count); } } while (count > 0); return memory.ToArray(); } } }? public byte[] DeflateDecompress(byte[] deflate) { using (DeflateStream stream = new DeflateStream(new MemoryStream(deflate), CompressionMode.Decompress)) { const int size = 4096; byte[] buffer = new byte[size]; using (MemoryStream memory = new MemoryStream()) { int count = 0; do { count = stream.Read(buffer, 0, size); if (count > 0) { memory.Write(buffer, 0, count); } } while (count > 0); return memory.ToArray(); } } }? private byte[] GetData(Stream sm) { byte[] realData = null; byte[] buffer = new byte[1024 * 8]; int dataLength = 0; do { dataLength = sm.Read(buffer, 0, buffer.Length); if (realData == null) { realData = new byte[dataLength]; Array.Copy(buffer, realData, dataLength); } else { int oldLength = realData.Length; Array.Resize<byte>(ref realData, realData.Length + dataLength); Array.Copy(buffer, 0, realData, oldLength, dataLength); }?? } while (dataLength > 0); //return (T)(object)buffer.Take(dataLength).ToArray(); return realData; }? private string GetHtml(Stream sm, string charset) { var data = GetData(sm); string newCharset = string.IsNullOrEmpty(charset) ? "utf-8" : charset; try { string r = Encoding.GetEncoding(newCharset).GetString(data); if (string.IsNullOrEmpty(charset)) { r = CheckEncoding(data, newCharset, r); } LogObject.WriteLine("==============================================\r\n"); LogObject.WriteLine(r); LogObject.WriteLine("=============================================="); LogObject.WriteLine("******************************分割*************************"); return r; } catch (Exception ex) { //get response error 503 ThrowException(509, ex.Message); return ""; } }? protected static Regex regCharset = new Regex("(?<=<meta.+?content\\=.+?charset\\=).+?(?=(\\\"|[\\s]))", RegexOptions.IgnoreCase); protected static Regex regCharset2 = new Regex("(?<=<meta[\\s]+charset=[\\\"]{0,1})[a-z0-9]+(?=[\\\"]{0,1})", RegexOptions.IgnoreCase); private string CheckEncoding(byte[] data, string currentCharset, string html) { string pageCharset = ""; if (regCharset.IsMatch(html)) { pageCharset = regCharset.Match(html).Value.Trim().ToLower(); } if (regCharset2.IsMatch(html)) { pageCharset = regCharset2.Match(html).Value.Trim().ToLower(); } if (pageCharset != currentCharset.Trim().ToLower()) { try { return Encoding.GetEncoding(pageCharset).GetString(data); } catch { } } return html; }? virtual protected X509CertificateCollection GetCertificates(Uri uri) { X509CertificateCollection certs = new X509CertificateCollection(); string host = uri.Host; for (int i = 0; i < 8; i++) { for (int j = 1; j <= 2; j++) { X509Store store = new X509Store((StoreName)(i + 1), (StoreLocation)j); store.Open(OpenFlags.MaxAllowed); foreach (var cert in store.Certificates) { Console.WriteLine(cert.Subject); if (cert.Subject.ToLower().Contains(host.ToLower())) { certs.Add(cert); } }? }? } return certs; }? virtual protected void ThrowException(int errorCode, string sourceText) { throw XQException.GetException(errorCode, sourceText); }? protected NameValueCollection GetQueryParameter(string url) { NameValueCollection pars = new NameValueCollection(); var paraString = url.Substring(url.IndexOf("?") + 1, url.Length - url.IndexOf("?") - 1).Split('&'); for (int i = 0; i < paraString.Length; i++) { var nv = paraString[i].Split('='); var name = nv[0]; var value = nv[1]; pars.Add(name, value); } return pars; } } QQ 討論組廣告群發(fā)工具（已開(kāi)發(fā)完成）索引

轉(zhuǎn)載于:https://www.cnblogs.com/Rolends/archive/2012/04/18/2455716.html

總結(jié)

以上是生活随笔為你收集整理的关于web自动化操作的分析和基类的实现的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。