How To Scrap Links from a Url in Asp.Net

How To Scrap Links from a Url in Asp.Net

Introduction

The main objective of this post is to demonstrate scraping of links from web pages(URL) using Asp.Net.
Generally, scraping of web pages is done with "HttpWebRequest" and "HttpWebResponse" method of C# in ASP.NET. However, it is observed that it becomes very difficult to fetch page data using "HttpWebRequest method. 

Background

In this post, I have created a demonstration web site. I will be scraping web site(URL) using Asp.NET.
Here, I have made use of third party tools "HtmlAgilityPack" to demonstrate this example.
You can get more information about this tool by visiting this link.

Code Snippet of Application

Aspx page:- 

<div width="100%">
        <asp:TextBox runat="server" ID="txtUrl" Width="300px"></asp:TextBox>
        <asp:Button runat="server" ID="btnscrap" Text="Scrap" OnClick="btnscrap_Click" />
        <br />
        <br />
        <asp:Label runat="server" ID="OutputLabel"></asp:Label>

    </div>


NameSpace Used Are:-

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using HtmlAgilityPack;

Note:- For NameSpace "HtmlAgilityPack" you need to add Reference in your project. You can Download  "HtmlAgilityPack" from :-

http://htmlagilitypack.codeplex.com/

cs page:- 

 protected void btnscrap_Click(object sender, EventArgs e)
        {
            var webGet = new HtmlWeb();
            var document = webGet.Load(txtUrl.Text.Trim());
            var aTags = document.DocumentNode.SelectNodes("//a");
            int counter = 1;
            if (aTags != null)
            {
                foreach (var aTag in aTags)
                {
                    if (aTag.Attributes.Contains("href"))
                    {
                        OutputLabel.Text += counter + ". " + aTag.InnerHtml + " - " + aTag.Attributes["href"].Value.ToString() + "\t" + "<br />";
                        counter++;
                    }
                }
            }

        }

Everything is done now.You just have to run the application.



Snaps:-

Enter Url And Click Download.

You will get the list of links on that url.

You can download sample code from below link:-


Scraping Example


Please give your feedback and comments on Post or You can mail me at samarthsolomon89@gmail.com ...!

Download Youtube Videos using Asp.Net

Download Youtube Videos using Asp.Net

Hi friends,

            In this post i will show you how to do download Youtube Videos using Asp.Net.


Step 1: On your aspx page paste the below code:- 

 <span>Enter YouTube URL:</span> <br />
    <asp:TextBox ID="txtYouTubeURL" runat="server" Text="" Width="450" />
    <br />
    <asp:Button ID="btnDownload" Text="Download" runat="server" OnClick="btnDownload_Click" />
    <br />
    <table>
        <tr>
            <td>
                <asp:LinkButton runat="server" ID="lnbFlV" Visible="false"></asp:LinkButton>
            </td>
        </tr>
        <tr>
            <td>
                <asp:LinkButton runat="server" ID="lnbMp4360" Visible="false"></asp:LinkButton>
            </td>
        </tr>
        <tr>
            <td>
                <asp:LinkButton runat="server" ID="lnbMp4480" Visible="false"></asp:LinkButton>
            </td>
        </tr>
        <tr>
            <td>
                <asp:LinkButton runat="server" ID="lnbMp4720" Visible="false"></asp:LinkButton>
            </td>
        </tr>
        <tr>
            <td>
                <asp:LinkButton runat="server" ID="lnbMp41080" Visible="false"></asp:LinkButton>
            </td>
        </tr>
    </table>

    <asp:Label ID="lblMessage" Text="" runat="server" />




NameSpace Used Are:-

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Net;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections.Specialized;

using System.ComponentModel;



Step 2: Now on your cs page paste below code



 protected void btnDownload_Click(object sender, EventArgs e)
{
    DownloadYouTubeVideo();
}




private void DownloadYouTubeVideo()
        {
            try
            {
                string sURL = txtYouTubeURL.Text.Trim();
                bool isDownloaded = false;

                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(sURL);
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                Stream responseStream = response.GetResponseStream();
                Encoding encode = System.Text.Encoding.GetEncoding("utf-8");
                StreamReader reader = new StreamReader(responseStream, encode);

                string pageSource = reader.ReadToEnd();
                pageSource = HttpUtility.HtmlDecode(pageSource);

                List<Uri> urls = GetVideoDownloadUrls(pageSource);

                string videoTitle = HttpUtility.HtmlDecode(GetVideoTitle(pageSource));

                foreach (Uri url in urls)
                {
                    if (url.AbsoluteUri.Contains("itag=5"))
                    {
                        lnbFlV.Text = "FLV 240p";
                        lnbFlV.PostBackUrl = url.AbsoluteUri + "&title=" + videoTitle;
                        lnbFlV.Visible = true;
                    }
                    if (url.AbsoluteUri.Contains("itag=18"))
                    {
                        lnbMp4360.Text = "MP4 360p";
                        lnbMp4360.PostBackUrl = url.AbsoluteUri + "&title=" + videoTitle;
                        lnbMp4360.Visible = true;
                    }
                    if (url.AbsoluteUri.Contains("itag=22"))
                    {
                        lnbMp4720.Text = "MP4 720p";
                        lnbMp4720.PostBackUrl = url.AbsoluteUri + "&title=" + videoTitle;
                        lnbMp4720.Visible = true;
                    }
                    if (url.AbsoluteUri.Contains("itag=137"))
                    {
                        lnbMp41080.Text = "MP4 1080p";
                        lnbMp41080.PostBackUrl = url.AbsoluteUri + "&title=" + videoTitle;
                        lnbMp41080.Visible = true;
                    }
                    if (url.AbsoluteUri.Contains("itag=135"))
                    {
                        lnbMp4480.Text = "MP4 480p";
                        lnbMp4480.PostBackUrl = url.AbsoluteUri + "&title=" + videoTitle;
                        lnbMp4480.Visible = true;
                    }

                }


            }
            catch (Exception e)
            {
                lblMessage.Text = "An error occurred while downloading video: " + e.Message;
            }
        }



private List<Uri> GetVideoDownloadUrls(string pageSource)
        {
            string pattern = "url=";
            string queryStringEnd = "&quality";

            List<Uri> finalURLs = new List<Uri>();
            List<string> urlList = Regex.Split(pageSource, pattern).ToList<string>();

            foreach (string url in urlList)
            {
                string sURL = HttpUtility.UrlDecode(url).Replace("\\u0026", "&");

                int index = sURL.IndexOf(queryStringEnd, StringComparison.Ordinal);
                if (index > 0)
                {
                    sURL = sURL.Substring(0, index).Replace("&sig=", "&signature=");
                    finalURLs.Add(new Uri(Uri.UnescapeDataString(sURL)));
                }
            }

            return finalURLs;
        }



private string GetVideoTitle(string pageSource)
        {
            int startIndex = pageSource.IndexOf("<title>");
            int endIndex = pageSource.IndexOf("</title>");

            if (startIndex > -1 && endIndex > -1)
            {
                string title = pageSource.Substring(startIndex + 7, endIndex - (startIndex + 7));
                return title;
            }

            return "Unknown";

        }

Everything is done now. Run the application.



Snaps:-

Enter Youtube Url And Click Download.

Now Links Will Appear Click on Quality You Want To Download

It will prompt to save the file

You can download sample code from below link:-


Youtube Video Download Example


Please give your feedback and comments...!