In our Website this error is come every 5 minutes. And I don't know from where its call. we need robots.txt without this error.
our application automatic call http://www.xyzName.com/content/images/thumbs/robots.txt And show below exception
System.Web.Routing.UrlRoutingModule does not implement IHttpHandlerFactory or IHttpHandler.
Description: An unhandled exception occurred during the execution of the current web request. Please review the stack trace for more information about the error and where it originated in the code.
This is the method which automatic generate the robots.txt file when this method is call. but the problem is from some where in application robots.txt call automatically.
public ActionResult RobotsTextFile()
{
//Default Code blocked by Nilesh
if (_storeContext.CurrentStore.Url.Contains("tk"))
{
const string newLine = "\r\n"; //Environment.NewLine
var sb = new StringBuilder();
sb.Append("User-agent: *");
sb.Append(newLine);
sb.Append("Disallow: /");
Response.ContentType = "text/plain";
Response.Write(sb.ToString());
}
else
{
var disallowPaths = new List<string>
{
"/bin/",
"/content/files/",
"/content/files/exportimport/",
"/country/getstatesbycountryid",
"/install",
"/setproductreviewhelpfulness",
};
var localizableDisallowPaths = new List<string>
{
"/addproducttocart/catalog/",
"/addproducttocart/details/",
"/backinstocksubscriptions/manage",
"/boards/forumsubscriptions",
"/boards/forumwatch",
"/boards/postedit",
"/boards/postdelete",
"/boards/postcreate",
"/boards/topicedit",
"/boards/topicdelete",
"/boards/topiccreate",
"/boards/topicmove",
"/boards/topicwatch",
"/cart",
"/checkout",
"/checkout/billingaddress",
"/checkout/completed",
"/checkout/confirm",
"/checkout/shippingaddress",
"/checkout/shippingmethod",
"/checkout/paymentinfo",
"/checkout/paymentmethod",
"/clearcomparelist",
"/compareproducts",
"/customer/avatar",
"/customer/activation",
"/customer/addresses",
"/customer/changepassword",
"/customer/checkusernameavailability",
"/customer/downloadableproducts",
"/customer/info",
"/deletepm",
"/emailwishlist",
"/inboxupdate",
"/newsletter/subscriptionactivation",
"/onepagecheckout",
"/order/history",
"/orderdetails",
"/passwordrecovery/confirm",
"/poll/vote",
"/privatemessages",
"/returnrequest",
"/returnrequest/history",
"/rewardpoints/history",
"/sendpm",
"/sentupdate",
"/shoppingcart/productdetails_attributechange",
"/subscribenewsletter",
"/topic/authenticate",
"/viewpm",
"/uploadfileproductattribute",
"/uploadfilecheckoutattribute",
"/wishlist",
};
const string newLine = "\r\n"; //Environment.NewLine
var sb = new StringBuilder();
sb.Append("User-agent: *");
sb.Append(newLine);
//sitemaps
if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
{
//URLs are localizable. Append SEO code
foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id))
{
sb.AppendFormat("Sitemap: {0}{1}/sitemap.xml", _storeContext.CurrentStore.Url, language.UniqueSeoCode);
sb.Append(newLine);
}
}
else
{
//localizable paths (without SEO code)
sb.AppendFormat("Sitemap: {0}sitemap.xml", _storeContext.CurrentStore.Url);
sb.Append(newLine);
}
//usual paths
foreach (var path in disallowPaths)
{
sb.AppendFormat("Disallow: {0}", path);
sb.Append(newLine);
}
//localizable paths (without SEO code)
foreach (var path in localizableDisallowPaths)
{
sb.AppendFormat("Disallow: {0}", path);
sb.Append(newLine);
}
if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
{
//URLs are localizable. Append SEO code
foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id))
{
foreach (var path in localizableDisallowPaths)
{
sb.AppendFormat("Disallow: {0}{1}", language.UniqueSeoCode, path);
sb.Append(newLine);
}
}
}
Response.ContentType = "text/plain";
Response.Write(sb.ToString());
}
return null;
}
And In RouteProvider we add below line to map the route.
routes.MapRoute("robots.txt","robots.txt",new { controller = "Common", action ="RobotsTextFile" },new[] { "Nop.Web.Controllers" });
It comes in every 5 minuets. we use amazon server for CDN, get Images from there.
Is there any possibility that amazon called this 'http://www.xyzName.com/content/images/thumbs/robots.txt' url?