update robots

Author: Nicholas Bergesen

Nick.RobotsParser/Nick.RobotsParser.csproj

@@ -5,21 +5,14 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <PackageId>Nick.RobotsParser</PackageId>
-    <Version>2.0.5</Version>
+    <Version>2.0.6</Version>
     <Authors>Nicholas Bergesen</Authors>
     <Company>Nicholas Bergesen</Company>
     <PackageReleaseNotes>
-	- Improve sitemap parsing robustness.
-	- Ignore sitemap parsing errors.
     </PackageReleaseNotes>
     <PackageTags>robots,parse robots,web crawling,robots.txt,web scraping,spider,sitemap,sitemap parse</PackageTags>
     <Description>
-      Library for working with robots.txt
-        - Parse robots.txt into Typed object.
-        - Lookup Allowed/Disallowed/Crawldelay based on User-Agent.
-        - Traverse sitemap in robots.txt for urls.
-
-      For More info see: https://github.com/nicholasbergesen/RobotsParser/blob/master/README.md
+		https://github.com/nicholasbergesen/RobotsParser/blob/master/README.md
     </Description>
   </PropertyGroup>
 

Nick.RobotsParser/RobotsParser.cs

@@ -1,10 +1,6 @@
-using System.IO.Compression;
-using System.Net.Http.Headers;
-using System.Runtime.CompilerServices;
+using System.Net.Http.Headers;
 using System.Text;
-using System.Transactions;
 using System.Xml;
-using System.Xml.Linq;
 using System.Xml.Serialization;
 
 namespace RobotsParser
@@ -14,15 +10,14 @@ public interface IRobots
         Task<bool> LoadRobotsFromUrl(string robotsUrl);
         Task<bool> LoadRobotsContent(string robotsContent);
         IReadOnlyList<Useragent> UserAgents { get; }
-        IEnumerable<string> Sitemaps { get; }
+        HashSet<string> Sitemaps { get; }
         IEnumerable<string> GetAllowedPaths(string userAgent = "*");
         IEnumerable<string> GetDisallowedPaths(string userAgent = "*");
         bool IsPathAllowed(string path, string userAgent = "*");
         bool IsPathDisallowed(string path, string userAgent = "*");
         int GetCrawlDelay(string userAgent = "*");
-        Task<IReadOnlyList<tSitemap>> GetSitemapIndexes(string sitemapUrl = null);
+        Task<IReadOnlyList<tSitemap>> GetSitemapIndexes(string? sitemapUrl = null);
         Task<IReadOnlyList<tUrl>> GetUrls(tSitemap tSitemap);
-        string? SitemapUrl { get; set; }
     }
 
     public class ProgressEventArgs : EventArgs
@@ -36,16 +31,13 @@ public ProgressEventArgs(string progressMessage)
 
     public class Robots : IRobots, IDisposable
     {
-        private readonly Uri _robotsUri;
-        private string? _robots;
+        private string? _robotsContent;
         private readonly HttpClient _client;
         private readonly bool _supressSitemapErrors;
 
         public event ProgressEventHandler? OnProgress;
         public delegate void ProgressEventHandler(object sender, ProgressEventArgs e);
 
-        public string? SitemapUrl { get; set; }
-
         public Robots(string userAgent, bool supressSitemapErrors = false)
         {
             _supressSitemapErrors = supressSitemapErrors;
@@ -82,14 +74,14 @@ private void RaiseOnProgress(string progressMessage)
 
         private async Task ParseRobots()
         {
-            if(_robots is null)
+            if(_robotsContent is null)
                 throw new RobotsNotloadedException();
 
             _userAgents = new List<Useragent>();
-            _sitemaps = new HashSet<string>();
+            _sitemaps ??= new HashSet<string>();
 
             string? line;
-            using (StringReader sr = new StringReader(_robots))
+            using (StringReader sr = new StringReader(_robotsContent))
             {
                 Useragent currentAgent = new Useragent("*");
                 while ((line = await sr.ReadLineAsync()) != null)
@@ -114,8 +106,6 @@ private async Task ParseRobots()
                         throw new Exception($"Unable to parse {line} in robots.txt");
                 }
             }
-
-            SitemapUrl = _sitemaps.FirstOrDefault();
         }
 
         #region Interface Methods
@@ -130,7 +120,7 @@ public async Task<bool> LoadRobotsFromUrl(string robotsUrl)
                 var response = await _client.GetAsync(robots);
                 response.EnsureSuccessStatusCode();
 
-                _robots = await response.Content.ReadAsStringAsync();
+                _robotsContent = await response.Content.ReadAsStringAsync();
             }
             catch (HttpRequestException e)
             {
@@ -144,7 +134,7 @@ public async Task<bool> LoadRobotsFromUrl(string robotsUrl)
 
         public async Task<bool> LoadRobotsContent(string robotsContent)
         {
-            _robots = robotsContent;
+            _robotsContent = robotsContent;
             await ParseRobots();
             return true;
         }
@@ -161,7 +151,7 @@ public IReadOnlyList<Useragent> UserAgents
         }
 
         private HashSet<string>? _sitemaps;
-        public IEnumerable<string> Sitemaps
+        public HashSet<string> Sitemaps
         {
             get
             {
@@ -199,37 +189,28 @@ public int GetCrawlDelay(string userAgent = "*") {
 
         public async Task<IReadOnlyList<tSitemap>> GetSitemapIndexes(string? sitemapUrl = null)
         {
-            SitemapUrl ??= sitemapUrl;
-
-            if (!string.IsNullOrEmpty(SitemapUrl))
+            if (!string.IsNullOrEmpty(sitemapUrl))
             {
                 _sitemaps ??= new HashSet<string>();
-                _sitemaps.Add(SitemapUrl);
+                _sitemaps.Add(sitemapUrl);
             }
 
             if (_sitemaps is null)
             {
-                if (_robots is null)
+                if (_robotsContent is null)
                     throw new RobotsNotloadedException("Please call LoadRobotsFromUrl, LoadRobotsContent or pass a sitemap url to GetSitemapIndexes.");
 
                 return new List<tSitemap>();
             }
 
-            //If not value given from user then go through _sitemaps.
-            if (string.IsNullOrEmpty(SitemapUrl))
-            {
-                List<tSitemap> sitemaps = new List<tSitemap>(100000);
-                if(_sitemaps.Count > 0)
-                {
-                    foreach (var sitemap in _sitemaps)
-                        sitemaps.AddRange(await GetSitemapsInternal(sitemap));
-                }
-                return sitemaps;
-            }
-            else 
+            List<tSitemap> sitemaps = new List<tSitemap>(100000);
+            if(_sitemaps.Any())
             {
-                return await GetSitemapsInternal(SitemapUrl);
+                foreach (var sitemap in _sitemaps)
+                    sitemaps.AddRange(await GetSitemapsInternal(sitemap));
             }
+
+            return sitemaps;
         }
 
         public async Task<IReadOnlyList<tUrl>> GetUrls(tSitemap tSitemap)