Skip to content

Commit 8dfd4e8

Browse files
committed
2025.10.4.0
YT YouTubeSettings: add property 'ParseLongUserTitle' YouTubeMediaContainerBase: concatenate artists SCrawler Bluesky: add saved posts downloading xHamster: temporarily disable the plugin
1 parent 1404afd commit 8dfd4e8

File tree

10 files changed

+256
-125
lines changed

10 files changed

+256
-125
lines changed

Changelog.md

Lines changed: 127 additions & 99 deletions
Large diffs are not rendered by default.

SCrawler.YouTube/Base/YouTubeSettings.vb

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,9 @@ Namespace API.YouTube.Base
268268
<Browsable(True), GridVisible, XMLVN({"Defaults"}, FileDateMode.None), Category("Defaults"), DisplayName("Add channel to file name"),
269269
Description("Add channel name before/after the file name")>
270270
Public ReadOnly Property FileAddChannelToFileName As XMLValue(Of FileDateMode)
271+
<Browsable(True), GridVisible, XMLVN({"Defaults"}, True), Category("Defaults"), DisplayName("Parse long user titles"),
272+
Description("Suitable for multiple artists")>
273+
Public ReadOnly Property ParseLongUserTitle As XMLValue(Of Boolean)
271274
#End Region
272275
#Region "Defaults ChannelsDownload"
273276
<Browsable(True), GridVisible, XMLVN({"Defaults", "Channels"}), Category("Defaults"), DisplayName("Default download tabs for channels"),

SCrawler.YouTube/My Project/AssemblyInfo.vb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
3232
' by using the '*' as shown below:
3333
' <Assembly: AssemblyVersion("1.0.*")>
3434

35-
<Assembly: AssemblyVersion("2025.8.30.0")>
36-
<Assembly: AssemblyFileVersion("2025.8.30.0")>
35+
<Assembly: AssemblyVersion("2025.10.4.0")>
36+
<Assembly: AssemblyFileVersion("2025.10.4.0")>
3737
<Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1716,12 +1716,20 @@ Namespace API.YouTube.Objects
17161716
If Not tmpPls.IsEmptyString Then PlaylistTitle = tmpPls
17171717
End If
17181718

1719+
Dim tmpTitle$
17191720
UserID = .Value("uploader_id")
17201721
UserTitle = TitleHtmlConverter.Invoke(.Value("uploader"))
17211722
If Not UserTitle.IsEmptyString Then
1722-
Dim tmpTitle$ = UserTitle.Replace("Topic", String.Empty).StringTrimEnd(" ", "-")
1723+
tmpTitle = UserTitle.Replace("Topic", String.Empty).StringTrimEnd(" ", "-")
17231724
If Not tmpTitle.IsEmptyString Then UserTitle = tmpTitle
17241725
End If
1726+
If MyYouTubeSettings.ParseLongUserTitle Or UserTitle.IsEmptyString Then
1727+
tmpTitle = TitleHtmlConverter.Invoke(.Value("artist"))
1728+
If Not tmpTitle.IsEmptyString Then
1729+
If Not UserTitle.IsEmptyString AndAlso Not tmpTitle.Contains(UserTitle) Then tmpTitle = $"{UserTitle}, {tmpTitle}"
1730+
UserTitle = ListAddList(Nothing, tmpTitle.Split(","), CType(Function(v$) v.StringTrim, Func(Of Object, Object)), EDP.ReturnValue).ListToString(" & ").IfNullOrEmpty(UserTitle)
1731+
End If
1732+
End If
17251733

17261734
Dim ext$ = IIf(IsMusic,
17271735
MyYouTubeSettings.DefaultAudioCodecMusic.Value.StringToLower,

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
3232
' by using the '*' as shown below:
3333
' <Assembly: AssemblyVersion("1.0.*")>
3434

35-
<Assembly: AssemblyVersion("2025.8.30.0")>
36-
<Assembly: AssemblyFileVersion("2025.8.30.0")>
35+
<Assembly: AssemblyVersion("2025.10.4.0")>
36+
<Assembly: AssemblyFileVersion("2025.10.4.0")>
3737
<Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Bluesky/SiteSettings.vb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Imports PersonalUtilities.Functions.RegularExpressions
1414
Imports PersonalUtilities.Tools.Web.Clients
1515
Imports PersonalUtilities.Tools.Web.Documents.JSON
1616
Namespace API.Bluesky
17-
<Manifest(BlueskySiteKey), SpecialForm(False)>
17+
<Manifest(BlueskySiteKey), SpecialForm(False), SavedPosts>
1818
Friend Class SiteSettings : Inherits SiteSettingsBase
1919
<PropertyOption(ControlText:="Cookies enabled", ControlToolTip:="If checked, cookies will be used in requests", IsAuth:=True), PXML, PClonable, HiddenControl>
2020
Friend ReadOnly Property CookiesEnabled As PropertyValue

SCrawler/API/Bluesky/UserData.vb

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,22 +79,33 @@ Namespace API.Bluesky
7979
Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
8080
Dim URL$ = String.Empty
8181
Try
82-
If ID.IsEmptyString Then GetProfileInfo(Token)
83-
If ID.IsEmptyString Then Throw New ArgumentNullException("ID", "ID is null")
82+
If Not IsSavedPosts And ID.IsEmptyString Then GetProfileInfo(Token)
83+
If Not IsSavedPosts And ID.IsEmptyString Then Throw New ArgumentNullException("ID", "ID is null")
8484
If UpdateToken() Then
8585
Dim nextCursor$ = String.Empty
8686
Dim c%
87-
URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter=posts_and_author_threads&includePins=false&limit=99"
88-
If Not Cursor.IsEmptyString Then URL &= $"&cursor={SymbolsConverter.ASCII.EncodeSymbolsOnly(Cursor)}"
87+
Dim n$(), p$()
88+
If IsSavedPosts Then
89+
URL = "https://bsky.social/xrpc/app.bsky.bookmark.getBookmarks"
90+
If Not Cursor.IsEmptyString Then URL &= $"?cursor={Cursor}"
91+
n = {"bookmarks"}
92+
p = {"item"}
93+
Else
94+
URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter=posts_and_author_threads&includePins=false&limit=99"
95+
If Not Cursor.IsEmptyString Then URL &= $"&cursor={SymbolsConverter.ASCII.EncodeSymbolsOnly(Cursor)}"
96+
n = {"feed"}
97+
p = {"post"}
98+
End If
8999
Dim r$ = Responser.GetResponse(URL)
90100
TokenUpdateCountReset()
91101
If Not r.IsEmptyString Then
92102
Using j As EContainer = JsonDocument.Parse(r)
93103
If j.ListExists Then
94-
With j("feed")
104+
nextCursor = j.Value("cursor")
105+
With j(n)
95106
If .ListExists Then
96107
For Each post As EContainer In .Self
97-
With post({"post"})
108+
With post(p)
98109
c = DefaultParser(.Self,, nextCursor)
99110
Select Case c
100111
Case CInt(DateResult.Skip) * -1 : Continue For
@@ -104,6 +115,8 @@ Namespace API.Bluesky
104115
If DownloadTopCount.HasValue AndAlso DownloadTopCount.Value <= _PostCount Then Exit Sub
105116
End With
106117
Next
118+
ElseIf IsSavedPosts Then
119+
nextCursor = String.Empty
107120
End If
108121
End With
109122
End If
@@ -126,7 +139,7 @@ Namespace API.Bluesky
126139
Optional ByVal CheckTempPosts As Boolean = True, Optional ByVal State As UStates = UStates.Unknown) As Integer
127140
Const exitReturn% = CInt(DateResult.Exit) * -1
128141
Const skipReturn% = CInt(DateResult.Skip) * -1
129-
Dim postID$, postDate$, __url$, __urlBase$, __txt$, __userId$
142+
Dim postID$, postDate$, __url$, __urlBase$, __txt$, __userId$, __postAuthor$
130143
Dim updateUrl As Boolean
131144
Dim c% = 0
132145
Dim m As UserMedia
@@ -138,11 +151,12 @@ Namespace API.Bluesky
138151
__urlBase = String.Empty
139152
__txt = String.Empty
140153
__userId = .Value({"author"}, "did")
154+
__postAuthor = String.Empty
141155
With .Item({"record"})
142156
If .ListExists Then
143157
'2025-01-28T02:42:12.415Z
144158
postDate = .Value("createdAt")
145-
NextCursor = postDate
159+
If Not IsSavedPosts Then NextCursor = postDate
146160
If CheckDateLimits Then
147161
Select Case CheckDatesLimit(postDate, DateProvider)
148162
Case DateResult.Skip : Return skipReturn 'Continue For
@@ -155,9 +169,10 @@ Namespace API.Bluesky
155169
If _TempPostsList.Contains(postID) Then Return exitReturn Else _TmpPosts2.Add(postID)
156170
End If
157171

158-
If ParseUserMediaOnly And Not ID.IsEmptyString And Not __userId.IsEmptyString And Not ID = __userId Then Return skipReturn
172+
If ParseUserMediaOnly And Not IsSavedPosts And Not ID.IsEmptyString And Not __userId.IsEmptyString And Not ID = __userId Then Return skipReturn
159173

160-
__urlBase = $"https://bsky.app/profile/{NameTrue}/post/{postID}"
174+
__postAuthor = e.Value({"author"}, "did")
175+
__urlBase = $"https://bsky.app/profile/{If(IsSavedPosts, __postAuthor, NameTrue)}/post/{postID}"
161176
End If
162177
End With
163178

@@ -190,7 +205,11 @@ Namespace API.Bluesky
190205
__url = d.Value("fullsize")
191206
If __url.IsEmptyString Then __url = d.Value({"image", "ref"}, "$link") : updateUrl = True
192207
If __url.IsEmptyString And SecondExtraction Then updateUrl = False : __url = e.Value({"embed"}, "thumb")
193-
If Not __url.IsEmptyString Then createMedia(__url, UTypes.Picture)
208+
If Not __url.IsEmptyString Then
209+
If updateUrl AndAlso Not __url.StartsWith("http") Then _
210+
__url = $"https://cdn.bsky.app/img/feed_fullsize/plain/{__postAuthor}/{__url}@jpeg"
211+
createMedia(__url, UTypes.Picture)
212+
End If
194213
Next
195214
End With
196215
End If

SCrawler/API/Xhamster/SiteSettings.vb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ Namespace API.Xhamster
7575
Return New UserData
7676
End Function
7777
Friend Overrides Function Available(ByVal What As ISiteSettings.Download, ByVal Silent As Boolean) As Boolean
78+
'TODELETE: xHamster disabled
79+
Return False
7880
If Settings.UseM3U8 AndAlso MyBase.Available(What, Silent) Then
7981
If What = ISiteSettings.Download.SavedPosts Then
8082
Return Responser.CookiesExists

SCrawler/API/Xhamster/UserData.vb

Lines changed: 78 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@
66
'
77
' This program is distributed in the hope that it will be useful,
88
' but WITHOUT ANY WARRANTY
9+
Imports System.Text
910
Imports System.Threading
10-
Imports SCrawler.API.Base
11-
Imports SCrawler.API.YouTube.Objects
11+
Imports PersonalUtilities.Functions.RegularExpressions
1212
Imports PersonalUtilities.Functions.XML
1313
Imports PersonalUtilities.Functions.XML.Base
14-
Imports PersonalUtilities.Functions.RegularExpressions
1514
Imports PersonalUtilities.Tools.Web.Clients
1615
Imports PersonalUtilities.Tools.Web.Documents.JSON
16+
Imports SCrawler.API.Base
17+
Imports SCrawler.API.YouTube.Objects
1718
Imports UTypes = SCrawler.API.Base.UserMedia.Types
1819
Namespace API.Xhamster
1920
Friend Class UserData : Inherits UserDataBase : Implements IPSite
@@ -564,15 +565,85 @@ Namespace API.Xhamster
564565
Return ErrorsDescriber.Execute(EDP.ReturnValue, ex, $"[{ToStringForLog()}]: API.Xhamster.GetM3U8({URL})", False)
565566
End Try
566567
End Function
567-
Private Overloads Function GetM3U8(ByRef m As UserMedia, ByVal j As EContainer, ByVal SpecFolder As String) As Boolean
568-
Dim node As EContainer = j({"xplayerSettings", "sources", "hls"})
568+
Private Overloads Function GetM3U8(ByRef m As UserMedia, ByVal j As EContainer, ByVal SpecFolder As String, Optional ByVal r As Integer = 0) As Boolean
569+
Const urlNode$ = "url"
570+
Dim node As EContainer = j({"xplayerSettings", "sources", If(r = 0, "hls", "standard")})
569571
If node.ListExists Then
570-
Dim url$ = node.GetNode({New NodeParams("url", True, True, True, True, 2)}).XmlIfNothingValue
571-
If Not url.IsEmptyString Then m.URL = url : m.Type = UTypes.m3u8 : Return True
572+
Dim url$ 'node.GetNode({New NodeParams("url", True, True, True, True, 2)}).XmlIfNothingValue
573+
Dim jn As EContainer, jn2 As EContainer
574+
Dim __getUrl As Func(Of EContainer, String) = Function(jj) If(jj.Contains(urlNode), Decipher_URL(jj.Value(urlNode)), String.Empty)
575+
url = __getUrl(node)
576+
If url.IsEmptyString Then
577+
For Each jn In node
578+
If jn.Contains(urlNode) Then
579+
url = __getUrl(jn)
580+
ElseIf jn.Count > 0 Then
581+
For Each jn2 In jn
582+
url = __getUrl(jn2)
583+
If Not url.IsEmptyString Then Exit For
584+
Next
585+
End If
586+
If Not url.IsEmptyString Then Exit For
587+
Next
588+
End If
589+
If Not url.IsEmptyString Then
590+
m.URL = url
591+
m.Type = UTypes.m3u8
592+
Return True
593+
End If
572594
End If
595+
If r = 0 Then Return GetM3U8(m, j, SpecFolder, r + 1)
573596
Return False
574597
End Function
575598
#End Region
599+
#Region "Decipher"
600+
'https://github.com/yt-dlp/yt-dlp/blob/5513036104ed9710f624c537fb3644b07a0680db/yt_dlp/extractor/xhamster.py#L146-L165
601+
Private Function Decipher_URL(ByVal Input As String) As String
602+
If Input.IsEmptyString Then Return String.Empty
603+
604+
Dim _XOR_KEY As Byte() = Encoding.ASCII.GetBytes("xh7999")
605+
Dim cipher_type$ = String.Empty
606+
Dim ciphertext$ = String.Empty
607+
608+
Try
609+
Dim decoded$ = Encoding.ASCII.GetString(Convert.FromBase64String(Input))
610+
Dim parts$() = decoded.Split(New Char() {"_"c}, 2)
611+
If parts.Length = 2 Then cipher_type = parts(0) : ciphertext = parts(1)
612+
Catch
613+
End Try
614+
615+
If cipher_type.IsEmptyString Or ciphertext.IsEmptyString Then Return String.Empty
616+
617+
If cipher_type = "xor" Then
618+
Dim ciphertextBytes() As Byte = Encoding.ASCII.GetBytes(ciphertext)
619+
Dim resultBytes(ciphertextBytes.Length - 1) As Byte
620+
For i% = 0 To ciphertextBytes.Length - 1
621+
resultBytes(i) = ciphertextBytes(i) Xor _XOR_KEY(i Mod _XOR_KEY.Length)
622+
Next
623+
Return Encoding.ASCII.GetString(resultBytes)
624+
End If
625+
626+
If cipher_type = "rot13" Then Return Decipher_URL_Rot13(ciphertext)
627+
628+
Return String.Empty
629+
End Function
630+
Private Function Decipher_URL_Rot13(ByVal Input As String) As String
631+
Dim result As New Text.StringBuilder(Input.Length)
632+
For Each c As Char In Input
633+
Dim offset%
634+
If c >= "a"c AndAlso c <= "z"c Then
635+
offset = Asc("a"c)
636+
result.Append(ChrW((Asc(c) - offset + 13) Mod 26 + offset))
637+
ElseIf c >= "A"c AndAlso c <= "Z"c Then
638+
offset = Asc("A"c)
639+
result.Append(ChrW((Asc(c) - offset + 13) Mod 26 + offset))
640+
Else
641+
result.Append(c)
642+
End If
643+
Next
644+
Return result.ToString
645+
End Function
646+
#End Region
576647
#Region "DownloadSingleObject"
577648
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
578649
_ContentList.Add(New UserMedia(Data.URL_BASE) With {.State = UserMedia.States.Missing})

SCrawler/My Project/AssemblyInfo.vb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
3232
' by using the '*' as shown below:
3333
' <Assembly: AssemblyVersion("1.0.*")>
3434

35-
<Assembly: AssemblyVersion("2025.9.1.0")>
36-
<Assembly: AssemblyFileVersion("2025.9.1.0")>
35+
<Assembly: AssemblyVersion("2025.10.4.0")>
36+
<Assembly: AssemblyFileVersion("2025.10.4.0")>
3737
<Assembly: NeutralResourcesLanguage("en")>

0 commit comments

Comments
 (0)