Replace more bs4.find functions with select

Author: Galarzaa90

docs/conf.py

@@ -200,4 +200,6 @@ def setup(app):
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None),
     'aiohttp': ('https://aiohttp.readthedocs.io/en/stable/', None),
+    'beautifulsoup4': ('https://www.crummy.com/software/BeautifulSoup/bs4/doc/', None),
+    'bs4': ('https://www.crummy.com/software/BeautifulSoup/bs4/doc/', None),
 }

tibiapy/parsers/bazaar.py

@@ -13,7 +13,7 @@
 from tibiapy.models import (AchievementEntry, AjaxPaginator, Auction, AuctionFilters, BestiaryEntry, BlessingEntry,
                             CharacterBazaar, CharmEntry, FamiliarEntry, Familiars, ItemEntry, ItemSummary, MountEntry,
                             Mounts, OutfitEntry, OutfitImage, Outfits, SalesArgument, SkillEntry)
-from tibiapy.utils import (convert_line_breaks, parse_form_data, parse_integer, parse_pagination,
+from tibiapy.utils import (clean_text, convert_line_breaks, get_rows, parse_form_data, parse_integer, parse_pagination,
                            parse_tibia_datetime, parse_tibiacom_content, try_enum)
 
 results_pattern = re.compile(r"Results: (\d+)")
@@ -264,8 +264,8 @@ def _parse_auction(cls, auction_row: bs4.Tag, auction_id=0) -> Auction:
 
         dates_containers = auction_row.select_one("div.ShortAuctionData")
         start_date_tag, end_date_tag, *_ = dates_containers.select("div.ShortAuctionDataValue")
-        builder.auction_start(parse_tibia_datetime(start_date_tag.text.replace("\xa0", " ")))
-        builder.auction_end(parse_tibia_datetime(end_date_tag.text.replace("\xa0", " ")))
+        builder.auction_start(parse_tibia_datetime(clean_text(start_date_tag)))
+        builder.auction_end(parse_tibia_datetime(clean_text(end_date_tag)))
         bids_container = auction_row.select_one("div.ShortAuctionDataBidRow")
         bid_tag = bids_container.select_one("div.ShortAuctionDataValue")
         bid_type_tag = bids_container.select("div.ShortAuctionDataLabel")[0]
@@ -324,7 +324,7 @@ def _parse_data_table(cls, table: bs4.Tag) -> Dict[str, str]:
         :class:`dict`
             A mapping containing the table's data.
         """
-        rows = table.select("tr")
+        rows = get_rows(table)
         data = {}
         for row in rows:
             name = row.select_one("span").text
@@ -343,7 +343,7 @@ def _parse_skills_table(cls, builder: AuctionDetailsBuilder, table):
         table: :class:`bs4.Tag`
             The table containing the character's skill.
         """
-        rows = table.select("tr")
+        rows = get_rows(table)
         skills = []
         for row in rows:
             cols = row.select("td")
@@ -364,7 +364,7 @@ def _parse_blessings_table(cls, builder: AuctionDetailsBuilder, table):
             The table containing the character's blessings.
         """
         table_content = table.select_one("table.TableContent")
-        _, *rows = table_content.select("tr")
+        _, *rows = get_rows(table_content)
         blessings = []
         for row in rows:
             cols = row.select("td")
@@ -389,7 +389,7 @@ def _parse_single_column_table(cls, table: bs4.Tag):
             A list with the contents of each row.
         """
         table_content = table.select("table.TableContent")[-1]
-        _, *rows = table_content.select("tr")
+        _, *rows = get_rows(table_content)
         ret = []
         for row in rows:
             col = row.select_one("td")
@@ -411,7 +411,7 @@ def _parse_charms_table(cls, builder: AuctionDetailsBuilder, table):
             The table containing the charms.
         """
         table_content = table.select_one("table.TableContent")
-        _, *rows = table_content.select("tr")
+        _, *rows = get_rows(table_content)
         charms = []
         for row in rows:
             cols = row.select("td")
@@ -434,7 +434,7 @@ def _parse_achievements_table(cls, builder: AuctionDetailsBuilder, table: bs4.Ta
             The table containing the achievements.
         """
         table_content = table.select_one("table.TableContent")
-        _, *rows = table_content.select("tr")
+        _, *rows = get_rows(table_content)
         achievements = []
         for row in rows:
             col = row.select_one("td")
@@ -457,7 +457,7 @@ def _parse_bestiary_table(cls, builder: AuctionDetailsBuilder, table: bs4.Tag, b
             The table containing the bestiary information.
         """
         table_content = table.select_one("table.TableContent")
-        _, *rows = table_content.select("tr")
+        _, *rows = get_rows(table_content)
         bestiary = []
         for row in rows:
             cols = row.select("td")
@@ -497,7 +497,7 @@ def _parse_general_table(cls, builder: AuctionDetailsBuilder, table):
         cls._parse_skills_table(builder, content_containers[1])
 
         additional_stats = cls._parse_data_table(content_containers[2])
-        builder.creation_date(parse_tibia_datetime(additional_stats.get("creation_date", "").replace("\xa0", " ")))
+        builder.creation_date(parse_tibia_datetime(clean_text(additional_stats.get("creation_date", ""))))
         builder.experience(parse_integer(additional_stats.get("experience", "0")))
         builder.gold(parse_integer(additional_stats.get("gold", "0")))
         builder.achievement_points(parse_integer(additional_stats.get("achievement_points", "0")))

tibiapy/parsers/character.py

@@ -10,7 +10,8 @@
 from tibiapy.errors import InvalidContent
 from tibiapy.models import (Achievement, Character, AccountBadge, AccountInformation, OtherCharacter, DeathParticipant,
                             Death, GuildMembership, CharacterHouse)
-from tibiapy.utils import (parse_popup, parse_tibia_date, parse_tibia_datetime, parse_tibiacom_content, split_list,
+from tibiapy.utils import (get_rows, parse_popup, parse_tibia_date, parse_tibia_datetime, parse_tibiacom_content,
+                           split_list,
                            try_enum, parse_link_info, clean_text, parse_integer)
 
 if TYPE_CHECKING:
@@ -100,8 +101,8 @@ def _parse_account_information(cls, rows):
             cols_raw = row.select("td")
             cols = [ele.text.strip() for ele in cols_raw]
             field, value = cols
-            field = field.replace("\xa0", "_").replace(" ", "_").replace(":", "").lower()
-            value = value.replace("\xa0", " ")
+            field = clean_text(field).replace(" ", "_").replace(":", "").lower()
+            value = clean_text(value)
             acc_info[field] = value
 
         created = parse_tibia_datetime(acc_info["created"])
@@ -127,7 +128,7 @@ def _parse_achievements(cls, rows):
             field, value = cols
             grade = str(field).count("achievement-grade-symbol")
             name = value.text.strip()
-            secret_image = value.find("img")
+            secret_image = value.select_one("img")
             secret = False
             if secret_image:
                 secret = True
@@ -247,7 +248,7 @@ def _parse_house_column(cls, builder: CharacterBuilder, column: bs4.Tag):
         m = house_regexp.search(house_text)
         paid_until = m.group(1)
         paid_until_date = parse_tibia_date(paid_until)
-        house_link_tag = column.find("a")
+        house_link_tag = column.select_one("a")
         house_link = parse_link_info(house_link_tag)
         builder.add_house(
             CharacterHouse(
@@ -264,7 +265,7 @@ def _parse_guild_column(cls, builder: CharacterBuilder, column: bs4.Tag):
         guild_link = column.select_one("a")
         value = clean_text(column)
         rank = value.split("of the")[0]
-        builder.guild_membership(GuildMembership(name=guild_link.text.replace("\xa0", " "), rank=rank.strip()))
+        builder.guild_membership(GuildMembership(name=clean_text(guild_link), rank=rank.strip()))
 
     @classmethod
     def _parse_deaths(cls, builder: CharacterBuilder, rows):
@@ -326,7 +327,7 @@ def _parse_killer(cls, killer):
         traded = False
         summon = None
         if traded_label in killer:
-            name = killer.replace("\xa0", " ").replace(traded_label, "").strip()
+            name = clean_text(killer).replace(traded_label, "").strip()
             traded = True
             player = True
 
@@ -359,7 +360,7 @@ def _parse_other_characters(cls, rows):
                 continue
 
             name, world, status, *__ = cols
-            _, *name = name.replace("\xa0", " ").split(" ")
+            _, *name = clean_text(name).split(" ")
             name = " ".join(name)
             traded = False
             if traded_label in name:
@@ -408,6 +409,6 @@ def _parse_tables(cls, parsed_content):
                 title = table.select_one("td").text.strip()
                 offset = 1
 
-            output[title] = table.select("tr")[offset:]
+            output[title] = get_rows(table)[offset:]
 
         return output

tibiapy/parsers/creature.py

@@ -34,7 +34,7 @@ class BoostedCreaturesParser:
 
     @classmethod
     def _parse_boosted_platform(cls, parsed_content: bs4.BeautifulSoup, tag_id: str):
-        img = parsed_content.find("img", attrs={"id": tag_id})
+        img = parsed_content.select_one(f"#{tag_id}")
         name = BOOSTED_ALT.sub("", img["title"]).strip()
         image_url = img["src"]
         identifier = image_url.split("/")[-1].replace(".gif", "")
@@ -94,13 +94,13 @@ def from_content(cls, content: str) -> BoostableBosses:
         """
         try:
             parsed_content = parse_tibiacom_content(content)
-            boosted_creature_table = parsed_content.find("div", {"class": "TableContainer"})
-            boosted_creature_text = boosted_creature_table.find("div", {"class": "Text"})
+            boosted_creature_table = parsed_content.select_one("div.TableContainer")
+            boosted_creature_text = boosted_creature_table.select_one("div.Text")
             if not boosted_creature_text or "Boosted" not in boosted_creature_text.text:
                 raise InvalidContent("content is not from the boostable bosses section.")
 
-            boosted_boss_tag = boosted_creature_table.find("b")
-            boosted_boss_image = boosted_creature_table.find("img")
+            boosted_boss_tag = boosted_creature_table.select_one("b")
+            boosted_boss_image = boosted_creature_table.select_one("img")
             image_url = urllib.parse.urlparse(boosted_boss_image["src"])
             boosted_boss = BossEntry(name=boosted_boss_tag.text,
                                      identifier=os.path.basename(image_url.path).replace(".gif", ""))
@@ -110,7 +110,7 @@ def from_content(cls, content: str) -> BoostableBosses:
             entries = []
             for entry_container in entries_container:
                 name = entry_container.text.strip()
-                image = entry_container.find("img")
+                image = entry_container.select_one("img")
                 image_url = urllib.parse.urlparse(image["src"])
                 identifier = os.path.basename(image_url.path).replace(".gif", "")
                 entries.append(BossEntry(name=name, identifier=identifier))
@@ -188,7 +188,7 @@ def from_content(cls, content: str) -> CreaturesSection:
             if not boosted_creature_text or "Boosted" not in boosted_creature_text.text:
                 raise InvalidContent("content is not from the creatures section.")
 
-            boosted_creature_link = boosted_creature_table.find("a")
+            boosted_creature_link = boosted_creature_table.select_one("a")
             url = urllib.parse.urlparse(boosted_creature_link["href"])
             query = urllib.parse.parse_qs(url.query)
             boosted_creature = CreatureEntry(name=boosted_creature_link.text, identifier=query["race"][0])
@@ -231,17 +231,17 @@ def from_content(cls, content: str) -> Optional[Creature]:
             pagination_container, content_container = (
                 parsed_content.find_all("div", style=lambda v: v and "position: relative" in v)
             )
-            title_container, description_container = content_container.find_all("div")
-            title = title_container.find("h2")
+            title_container, description_container = content_container.select("div")
+            title = title_container.select_one("h2")
             name = title.text.strip()
 
-            img = title_container.find("img")
+            img = title_container.select_one("img")
             img_url = img["src"]
             race = img_url.split("/")[-1].replace(".gif", "")
             builder = CreatureBuilder().name(name).identifier(race)
 
             convert_line_breaks(description_container)
-            paragraph_tags = description_container.find_all("p")
+            paragraph_tags = description_container.select("p")
             paragraphs = [p.text for p in paragraph_tags]
             builder.description("\n".join(paragraphs[:-2]).strip())
             hp_text = paragraphs[-2]

tibiapy/parsers/forum.py

@@ -12,7 +12,8 @@
 from tibiapy.models import (AnnouncementEntry, BoardEntry, CMPost, CMPostArchive, ForumAnnouncement, ForumAuthor,
                             ForumBoard, ForumEmoticon, ForumPost, ForumSection, ForumThread, GuildMembership, LastPost,
                             ThreadEntry)
-from tibiapy.utils import (convert_line_breaks, parse_form_data, parse_integer, parse_link_info, parse_pagination,
+from tibiapy.utils import (clean_text, convert_line_breaks, get_rows, parse_form_data, parse_integer, parse_link_info,
+                           parse_pagination,
                            parse_tables_map, parse_tibia_datetime, parse_tibia_forum_datetime, parse_tibiacom_content,
                            split_list, try_enum)
 
@@ -83,16 +84,16 @@ def from_content(cls, content: str) -> CMPostArchive:
 
         inner_table_container = table.select_one("div.InnerTableContainer")
         inner_table = inner_table_container.select_one("table")
-        inner_table_rows = inner_table.select("tr")
+        inner_table_rows = get_rows(inner_table)
         inner_table_rows = [e for e in inner_table_rows if e.parent == inner_table]
         table_content = inner_table_container.select_one("table.TableContent")
 
-        header_row, *rows = table_content.select("tr")
+        header_row, *rows = get_rows(table_content)
 
         for row in rows:
             columns = row.select("td")
             date_column = columns[0]
-            date = parse_tibia_datetime(date_column.text.replace("\xa0", " "))
+            date = parse_tibia_datetime(clean_text(date_column))
             board_thread_column = columns[1]
             convert_line_breaks(board_thread_column)
             board, thread = board_thread_column.text.splitlines()
@@ -414,7 +415,7 @@ def from_content(cls, content: str) -> Optional[ForumBoard]:
         builder.current_page(pages)
         builder.total_pages(total)
 
-        *thread_rows, times_row = tables[-1].select("tr")
+        *thread_rows, times_row = get_rows(tables[-1])
         for thread_row in thread_rows[1:]:
             columns = thread_row.select("td")
             entry = cls._parse_thread_row(columns)
@@ -424,7 +425,7 @@ def from_content(cls, content: str) -> Optional[ForumBoard]:
             builder.add_entry(entry)
 
         if len(tables) > 1:
-            announcement_rows = tables[0].select("tr")
+            announcement_rows = get_rows(tables[0])
             for announcement_row in announcement_rows[1:]:
                 author_link, title_link = announcement_row.select("a")
                 author = author_link.text.strip()
@@ -724,13 +725,13 @@ def _parse_column(cls, last_post_column, offset=1):
         permalink_tag = last_post_info.select_one("a")
         permalink_info = parse_link_info(permalink_tag)
         post_id = int(permalink_info["query"]["postid"])
-        date_text = last_post_info.text.replace("\xa0", " ").strip()
+        date_text = clean_text(last_post_info)
         last_post_date = parse_tibia_forum_datetime(date_text, offset)
 
         last_post_author_tag = last_post_column.select_one("font")
         author_link = last_post_author_tag.select_one("a")
         deleted = author_link is None
-        author = last_post_author_tag.text.replace("by", "", 1).replace("\xa0", " ").strip()
+        author = clean_text(last_post_author_tag).replace("by", "", 1)
         traded = False
         if "(traded)" in author:
             author = author.replace("(traded)", "").strip()