Update robots.ts

Author: rivea0

app/robots.ts

@@ -31,6 +31,7 @@ export default function robots(): MetadataRoute.Robots {
         'Cotoyogi',
         'Crawlspace',
         'Datenbank Crawler',
+        'DeepSeekBot',
         'Devin',
         'Diffbot',
         'DuckAssistBot',
@@ -44,6 +45,8 @@ export default function robots(): MetadataRoute.Robots {
         'Gemini-Deep-Research',
         'Google-CloudVertexBot',
         'Google-Extended',
+        'Google-Firebase',
+        'Google-NotebookLM',
         'GoogleAgent-Mariner',
         'GoogleOther',
         'GoogleOther-Image',
@@ -61,6 +64,7 @@ export default function robots(): MetadataRoute.Robots {
         'Meta-ExternalAgent',
         'meta-externalfetcher',
         'Meta-ExternalFetcher',
+        'meta-webindexer',
         'MistralAI-User',
         'MistralAI-User/1.0',
         'MyCentralAIScraperBot',
@@ -69,6 +73,7 @@ export default function robots(): MetadataRoute.Robots {
         'OAI-SearchBot',
         'omgili',
         'omgilibot',
+        'OpenAI',
         'Operator',
         'PanguBot',
         'Panscient',
@@ -89,8 +94,10 @@ export default function robots(): MetadataRoute.Robots {
         'SemrushBot-OCOB',
         'SemrushBot-SI',
         'SemrushBot-SWA',
+        'ShapBot',
         'Sidetrade indexer bot',
         'SummalyBot',
+        'TerraCotta',
         'Thinkbot',
         'TikTokSpider',
         'Timpibot',