I'm pretty sure it isn't so. If I set Encoding to (for example) UTF32 the
WebClient throws an exception. And if I have a page with an UTF8 character
(a page that in the WebRequest IS correctly shown as UTF8 page) and I don't
set the Encoder I receive a wrong String.
--- bye
Try this code. It attemps to get the CharacterSet in various ways and falls back to UTF-8. Checking for ContentEncoding may not be necessary as I have yet to see it specified. The code is a bit of cut and paste and you may have to tweak it to get it running.
public string DownloadPage(url)
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
{
using (Stream s = resp.GetResponseStream())
{
buffer = ReadStream(s);
}
string pageEncoding = "";
Encoding e = Encoding.UTF8;
if (resp.ContentEncoding != "")
pageEncoding = resp.ContentEncoding;
else if (resp.CharacterSet != "")
pageEncoding = resp.CharacterSet;
else if (resp.ContentType != "")
pageEncoding = GetCharacterSet(resp.ContentType);
if(pageEncoding == "")
pageEncoding = GetCharacterSet(buffer);
if (pageEncoding != "")
{
try
{
e = Encoding.GetEncoding(pageEncoding);
}
catch
{
MessageBox.Show("Invalid encoding: " + pageEncoding);
}
}
string data = e.GetString(buffer);
Status = "";
return data;
}
}
private string GetCharacterSet(string s)
{
s = s.ToUpper();
int start = s.LastIndexOf("CHARSET");
if (start == -1)
return "";
start = s.IndexOf("=", start);
if (start == -1)
return "";
start++;
s = s.Substring(start).Trim();
int end = s.Length;
int i = s.IndexOf(";");
if (i != -1)
end = i;
i = s.IndexOf("\"");
if (i != -1 && i < end)
end = i;
i = s.IndexOf("'");
if (i != -1 && i < end)
end = i;
i = s.IndexOf("/");
if (i != -1 && i < end)
end = i;
return s.Substring(0, end).Trim();
}
private string GetCharacterSet(byte[] data)
{
string s = Encoding.Default.GetString(data);
return GetCharacterSet(s);
}
private byte[] ReadStream(Stream s)
{
try
{
byte[] buffer = new byte[8096];
using (MemoryStream ms = new MemoryStream())
{
while (true)
{
int read = s.Read(buffer, 0, buffer.Length);
if (read <= 0)
{
CurLength = 0;
return ms.ToArray();
}
ms.Write(buffer, 0, read);
CurLength = ms.Length;
}
}
}
catch (Exception ex)
{
return null;
}
}
