Tag: sitemap

I wrote a small sitemap-checker.aspx, the goal of this code is to find the pages, that respond with the codes other than 200. I’ve only put the C# part here for the review:

    void Start_Click(object sender, EventArgs e)
    {
        CheckUrls(UrlTextBox.Text);
    }

    private void CheckUrls(string url)
    {
        try
        {
            var xmlDoc = LoadXml(url);
            var urlNodes = xmlDoc.GetElementsByTagName("url");
            var numberOfNodes = urlNodes.Count;
            var index = 0;
            
            Log("### Total nodes: " + numberOfNodes+ "<br>");
            Log("Here is the list of URLs, with the status code other than 200: <br>");
            foreach (XmlNode urlNode in urlNodes)
            {
                CheckUrlNode(urlNode, xmlDoc);
                index++;

                if(index % 10 == 0)
                {
                    Log("<span style='color:red'>" + index + " nodes of " + numberOfNodes + " are finished, please wait...</span>");
                }
            };
            Log("### Sitemap check finished. Please analyze the reported urls<br><br>");
        }
        catch (Exception e)
        {
            Log("Some problems were encountered while checking the sitemap<br>");
            Log(e.Message + "<br>");
            Log(e.StackTrace + "<br>");
        }
    }

    private XmlDocument LoadXml(string url)
    {
        var m_strFilePath = url;
        string xmlStr;
        using (var wc = new WebClient())
        {
            xmlStr = wc.DownloadString(m_strFilePath);
        }
        var xmlDoc = new XmlDocument();
        xmlDoc.LoadXml(xmlStr);
        return xmlDoc;
    }

    private void CheckUrlNode(XmlNode xmlNode, XmlDocument xmlDoc)
    {
        var urlList = new List<string>() { xmlNode("loc").InnerText };

        var nsmgr = new XmlNamespaceManager(xmlDoc.NameTable);
        nsmgr.AddNamespace("xhtml", "http://www.w3.org/1999/xhtml");

        var hrefLangUrls = xmlNode
            .SelectNodes("xhtml:link", nsmgr)
            .Cast<XmlNode>()
            .Select(x => x.Attributes("href").Value);

        urlList.AddRange(hrefLangUrls);

        foreach (var url in urlList)
        {
            string logMessage;
            if (!CheckUrl(url, out logMessage))
            {
                Log(logMessage);
            }
        }
    }

    private bool CheckUrl(string url, out string logMessage)
    {
        logMessage = null;
        try
        {
            var request = HttpWebRequest.Create(url) as HttpWebRequest;
            request.Timeout = 5000; //set the timeout to 5 seconds to keep the user from waiting too long for the page to load
            request.Method = "HEAD"; //Get only the header information -- no need to download any content

            using (var response = request.GetResponse() as HttpWebResponse)
            {
                int statusCode = (int)response.StatusCode;
                if (statusCode == 200) //Good requests
                {
                    return true;
                }
                else
                {
                    logMessage = String.Format("<a href='{0}'>{0}</a>, status code: {1}", url, statusCode);
                    return false;
                }
            }
        }
        catch (WebException ex)
        {
            logMessage = String.Format("<a href='{0}'>{0}</a>, status: {1}", url, ex.Status);
        }
        catch (Exception ex)
        {
            logMessage = String.Format("Could not test url {0}, exeption message: {1}", url, ex.Message);
        }

        return false;
    }