Abot.Core.WebContentExtractor.GetRawData(System.Net.WebResponse)

Here are the examples of the csharp api class Abot.Core.WebContentExtractor.GetRawData(System.Net.WebResponse) taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

1 Example 7

1. Example

Project: abot
Source File: WebContentExtractor.cs
public virtual PageContent GetContent(WebResponse response)
        {
            using (MemoryStream memoryStream = GetRawData(response))
            {
                String charset = GetCharsetFromHeaders(response);

                if (charset == null) {
                    memoryStream.Seek(0, SeekOrigin.Begin);

                    // Do not wrap in closing statement to prevent closing of this stream.
                    StreamReader srr = new StreamReader(memoryStream, Encoding.ASCII);
                    String body = srr.ReadToEnd();
                    charset = GetCharsetFromBody(body);
                }
                memoryStream.Seek(0, SeekOrigin.Begin);

                charset = CleanCharset(charset);
                Encoding e = GetEncoding(charset);
                string content = "";
                using (StreamReader sr = new StreamReader(memoryStream, e))
                {
                    content = sr.ReadToEnd();
                }

                PageContent pageContent = new PageContent();
                pageContent.Bytes = memoryStream.ToArray();
                pageContent.Charset = charset;
                pageContent.Encoding = e;
                pageContent.Text = content;

                return pageContent;
            }
        }