Downloads a web page from the Internet and returns a string. : HttpWebRequest « Network « C# / C Sharp

Home
C# / C Sharp
1.2D Graphics
2.Class Interface
3.Collections Data Structure
4.Components
5.Data Types
6.Database ADO.net
7.Date Time
8.Design Patterns
9.Development Class
10.Event
11.File Stream
12.Generics
13.GUI Windows Form
14.Internationalization I18N
15.Language Basics
16.LINQ
17.Network
18.Office
19.Reflection
20.Regular Expressions
21.Security
22.Services Event
23.Thread
24.Web Services
25.Windows
26.Windows Presentation Foundation
27.XML
28.XML LINQ
C# / C Sharp » Network » HttpWebRequest 




Downloads a web page from the Internet and returns a string.
    
using System;
using System.IO;
using System.Net.Mail;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Configuration;
using System.Globalization;
using System.Web;
using System.Web.Configuration;
using System.Threading;
using System.Reflection;
using System.Collections;
using System.Xml;
using System.Net;
using System.Web.Caching;

namespace BlogEngine.Core
{
  /// <summary>
  /// Utilities for the entire solution to use.
  /// </summary>
  public static class Utils
  {

    /// <summary>
    /// Downloads a web page from the Internet and returns a string. .
    /// </summary>
    /// <param name="url">The URL to download from.</param>
    /// <returns>The HTML or null if the URL isn't valid.</returns>
    public static string DownloadWebPage(Uri url)
    {
      try
      {
        using (WebClient client = new WebClient())
        {
          client.UseDefaultCredentials = true;
          client.Headers.Add(System.Net.HttpRequestHeader.UserAgent, "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1;)");
          using (StreamReader reader = new StreamReader(client.OpenRead(url)))
          {
            return reader.ReadToEnd();
          }
        }
      }
      catch (WebException)
      {
        return null;
      }
    }


    private static XmlDocument LoadDocument(Uri url, Uri xmlUrl)
    {
      XmlDocument doc = new XmlDocument();

      try
      {
        if (url.IsAbsoluteUri)
        {
          doc.Load(xmlUrl.ToString());
        }
        else
        {
          string absoluteUrl = null;
          if (!url.ToString().StartsWith("/"))
            absoluteUrl = (url + xmlUrl.ToString());
          else
            absoluteUrl = url.Scheme + "://" + url.Authority + xmlUrl;

          doc.Load(absoluteUrl);
        }
      }
      catch (Exception)
      {
        return null;
      }

      return doc;
    }

    private const string PATTERN = "<head.*<link( [^>]*title=\"{0}\"[^>]*)>.*</head>";
    private static readonly Regex HREF = new Regex("href=\"(.*)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);

    /// <summary>
    /// Finds semantic links in a given HTML document.
    /// </summary>
    /// <param name="type">The type of link. Could be foaf, apml or sioc.</param>
    /// <param name="html">The HTML to look through.</param>
    /// <returns></returns>
    public static List<Uri> FindLinks(string type, string html)
    {
      MatchCollection matches = Regex.Matches(html, string.Format(PATTERN, type), RegexOptions.IgnoreCase | RegexOptions.Singleline);
      List<Uri> urls = new List<Uri>();

      foreach (Match match in matches)
      {
        if (match.Groups.Count == 2)
        {
          string link = match.Groups[1].Value;
          Match hrefMatch = HREF.Match(link);

          if (hrefMatch.Groups.Count == 2)
          {
            Uri url;
            string value = hrefMatch.Groups[1].Value;
            if (Uri.TryCreate(value, UriKind.Absolute, out url))
            {
              urls.Add(url);
            }
          }
        }
      }

      return urls;
    }
  }
}

   
    
    
    
  














Related examples in the same category
1.Begins an asynchronous request for a Stream object to use to write data.
2.Gets or sets the value of the Connection HTTP header.
3.Gets or sets the cookies associated with the request.
4.Performs online file transfer operations.
java2s.com  | Contact Us | Privacy Policy
Copyright 2009 - 12 Demo Source and Support. All rights reserved.
All other trademarks are property of their respective owners.