Skip to content

Commit 97fbfce

Browse files
kynraiLissy93
kynrai
and
Lissy93
authored
RF: block lists to check and test resolvers (#48)
* RF: block lists to check and test resolvers - Move block list logic to checks - Create IP resolver interfaces - Inject resolvers - Add code cov file for code that just calls stdlib * Formatting --------- Co-authored-by: Alicia Sykes <alicia@omg.lol>
1 parent 9bc9d1f commit 97fbfce

File tree

10 files changed

+221
-156
lines changed

10 files changed

+221
-156
lines changed

‎checks/block_lists.go‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package checks
2+
3+
import (
4+
"context"
5+
"net"
6+
"slices"
7+
"sort"
8+
"sync"
9+
"time"
10+
11+
"github.com/xray-web/web-check-api/checks/clients/ip"
12+
)
13+
14+
type dnsServer struct {
15+
Name string
16+
IP string
17+
}
18+
19+
var DNS_SERVERS = []dnsServer{
20+
{Name: "AdGuard", IP: "176.103.130.130"},
21+
{Name: "AdGuard Family", IP: "176.103.130.132"},
22+
{Name: "CleanBrowsing Adult", IP: "185.228.168.10"},
23+
{Name: "CleanBrowsing Family", IP: "185.228.168.168"},
24+
{Name: "CleanBrowsing Security", IP: "185.228.168.9"},
25+
{Name: "CloudFlare", IP: "1.1.1.1"},
26+
{Name: "CloudFlare Family", IP: "1.1.1.3"},
27+
{Name: "Comodo Secure", IP: "8.26.56.26"},
28+
{Name: "Google DNS", IP: "8.8.8.8"},
29+
{Name: "Neustar Family", IP: "156.154.70.3"},
30+
{Name: "Neustar Protection", IP: "156.154.70.2"},
31+
{Name: "Norton Family", IP: "199.85.126.20"},
32+
{Name: "OpenDNS", IP: "208.67.222.222"},
33+
{Name: "OpenDNS Family", IP: "208.67.222.123"},
34+
{Name: "Quad9", IP: "9.9.9.9"},
35+
{Name: "Yandex Family", IP: "77.88.8.7"},
36+
{Name: "Yandex Safe", IP: "77.88.8.88"},
37+
}
38+
39+
var knownBlockIPs = []string{
40+
"146.112.61.106",
41+
"185.228.168.10",
42+
"8.26.56.26",
43+
"9.9.9.9",
44+
"208.69.38.170",
45+
"208.69.39.170",
46+
"208.67.222.222",
47+
"208.67.222.123",
48+
"199.85.126.10",
49+
"199.85.126.20",
50+
"156.154.70.22",
51+
"77.88.8.7",
52+
"77.88.8.8",
53+
"::1",
54+
"2a02:6b8::feed:0ff",
55+
"2a02:6b8::feed:bad",
56+
"2a02:6b8::feed:a11",
57+
"2620:119:35::35",
58+
"2620:119:53::53",
59+
"2606:4700:4700::1111",
60+
"2606:4700:4700::1001",
61+
"2001:4860:4860::8888",
62+
"2a0d:2a00:1::",
63+
"2a0d:2a00:2::",
64+
}
65+
66+
type Blocklist struct {
67+
Server string `json:"server"`
68+
ServerIP string `json:"serverIp"`
69+
IsBlocked bool `json:"isBlocked"`
70+
}
71+
72+
type BlockList struct {
73+
lookup ip.DNSLookup
74+
}
75+
76+
func NewBlockList(lookup ip.DNSLookup) *BlockList {
77+
return &BlockList{lookup: lookup}
78+
}
79+
80+
func (b *BlockList) domainBlocked(ctx context.Context, domain, serverIP string) bool {
81+
ips, err := b.lookup.DNSLookupIP(ctx, "ip4", domain, serverIP)
82+
if err != nil {
83+
// if there's an error, consider it not blocked
84+
// TODO: return more detailed errors for each server
85+
return false
86+
}
87+
88+
return slices.ContainsFunc(ips, func(ip net.IP) bool {
89+
return slices.Contains(knownBlockIPs, ip.String())
90+
})
91+
}
92+
93+
func (b *BlockList) BlockedServers(ctx context.Context, domain string) []Blocklist {
94+
var lock sync.Mutex
95+
var wg sync.WaitGroup
96+
limit := make(chan struct{}, 5)
97+
98+
var results []Blocklist
99+
100+
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
101+
defer cancel()
102+
103+
for _, server := range DNS_SERVERS {
104+
wg.Add(1)
105+
go func(server dnsServer) {
106+
limit <- struct{}{}
107+
defer func() {
108+
<-limit
109+
wg.Done()
110+
}()
111+
112+
isBlocked := b.domainBlocked(ctx, domain, server.IP)
113+
lock.Lock()
114+
defer lock.Unlock()
115+
results = append(results, Blocklist{
116+
Server: server.Name,
117+
ServerIP: server.IP,
118+
IsBlocked: isBlocked,
119+
})
120+
}(server)
121+
}
122+
wg.Wait()
123+
124+
sort.Slice(results, func(i, j int) bool {
125+
return results[i].Server < results[j].Server
126+
})
127+
return results
128+
}

‎checks/block_lists_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package checks
2+
3+
import (
4+
"context"
5+
"net"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
"github.com/xray-web/web-check-api/checks/clients/ip"
10+
)
11+
12+
func TestBlockList(t *testing.T) {
13+
t.Parallel()
14+
15+
t.Run("blocked IP", func(t *testing.T) {
16+
t.Parallel()
17+
18+
dnsLookup := ip.DNSLookupFunc(func(ctx context.Context, network, host, dns string) ([]net.IP, error) {
19+
return []net.IP{net.ParseIP("146.112.61.106")}, nil
20+
})
21+
list := NewBlockList(dnsLookup).BlockedServers(context.Background(), "example.com")
22+
assert.Contains(t, list, Blocklist{Server: "AdGuard", ServerIP: "176.103.130.130", IsBlocked: true})
23+
})
24+
}

‎checks/checks.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import (
44
"net/http"
55
"time"
66

7+
"github.com/xray-web/web-check-api/checks/clients/ip"
78
"github.com/xray-web/web-check-api/checks/store/legacyrank"
89
)
910

1011
type Checks struct {
12+
BlockList *BlockList
1113
Carbon *Carbon
1214
Headers *Headers
1315
IpAddress *Ip
@@ -23,8 +25,9 @@ func NewChecks() *Checks {
2325
Timeout: 5 * time.Second,
2426
}
2527
return &Checks{
28+
BlockList: NewBlockList(&ip.NetDNSLookup{}),
2629
Carbon: NewCarbon(client),
27-
Headers: NewHeaders(client),
30+
Headers: NewHeaders(client),
2831
IpAddress: NewIp(NewNetIp()),
2932
LegacyRank: NewLegacyRank(legacyrank.NewInMemoryStore()),
3033
LinkedPages: NewLinkedPages(client),

‎checks/clients/ip/ip.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package ip
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net"
7+
"time"
8+
)
9+
10+
type Lookup interface {
11+
LookupIP(ctx context.Context, network string, host string) ([]net.IP, error)
12+
}
13+
14+
type LookupFunc func(ctx context.Context, network string, host string) ([]net.IP, error)
15+
16+
func (fn LookupFunc) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) {
17+
return fn(ctx, network, host)
18+
}
19+
20+
// NetLookup is a client for looking up IP addresses using a net.Resolver.
21+
type NetLookup struct{}
22+
23+
func (l *NetLookup) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) {
24+
netResolver := &net.Resolver{
25+
PreferGo: true,
26+
}
27+
return netResolver.LookupIP(ctx, network, host)
28+
}
29+
30+
type DNSLookup interface {
31+
DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error)
32+
}
33+
34+
type DNSLookupFunc func(ctx context.Context, network, host, dns string) ([]net.IP, error)
35+
36+
func (fn DNSLookupFunc) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) {
37+
return fn(ctx, network, host, dns)
38+
}
39+
40+
// DNSLookup is a client for looking up IP addresses with a custom DNS server.
41+
type NetDNSLookup struct{}
42+
43+
func (l *NetDNSLookup) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) {
44+
netResolver := &net.Resolver{
45+
PreferGo: true,
46+
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
47+
d := net.Dialer{
48+
Timeout: 3 * time.Second,
49+
}
50+
return d.DialContext(ctx, network, fmt.Sprintf("%s:%d", dns, 53))
51+
},
52+
}
53+
return netResolver.LookupIP(ctx, network, host)
54+
}

‎codecov.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ignore:
2+
- checks/clients/ip/ip.go # this contains go std lib code wrapped for interfaces, not worth testing

‎handlers/block_lists.go‎

Lines changed: 5 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -1,143 +1,19 @@
11
package handlers
22

33
import (
4-
"context"
5-
"encoding/json"
6-
"net"
74
"net/http"
8-
"slices"
9-
"sort"
10-
"sync"
11-
"time"
12-
)
13-
14-
type dnsServer struct {
15-
Name string
16-
IP string
17-
}
18-
19-
var DNS_SERVERS = []dnsServer{
20-
{Name: "AdGuard", IP: "176.103.130.130"},
21-
{Name: "AdGuard Family", IP: "176.103.130.132"},
22-
{Name: "CleanBrowsing Adult", IP: "185.228.168.10"},
23-
{Name: "CleanBrowsing Family", IP: "185.228.168.168"},
24-
{Name: "CleanBrowsing Security", IP: "185.228.168.9"},
25-
{Name: "CloudFlare", IP: "1.1.1.1"},
26-
{Name: "CloudFlare Family", IP: "1.1.1.3"},
27-
{Name: "Comodo Secure", IP: "8.26.56.26"},
28-
{Name: "Google DNS", IP: "8.8.8.8"},
29-
{Name: "Neustar Family", IP: "156.154.70.3"},
30-
{Name: "Neustar Protection", IP: "156.154.70.2"},
31-
{Name: "Norton Family", IP: "199.85.126.20"},
32-
{Name: "OpenDNS", IP: "208.67.222.222"},
33-
{Name: "OpenDNS Family", IP: "208.67.222.123"},
34-
{Name: "Quad9", IP: "9.9.9.9"},
35-
{Name: "Yandex Family", IP: "77.88.8.7"},
36-
{Name: "Yandex Safe", IP: "77.88.8.88"},
37-
}
38-
39-
var knownBlockIPs = []string{
40-
"146.112.61.106",
41-
"185.228.168.10",
42-
"8.26.56.26",
43-
"9.9.9.9",
44-
"208.69.38.170",
45-
"208.69.39.170",
46-
"208.67.222.222",
47-
"208.67.222.123",
48-
"199.85.126.10",
49-
"199.85.126.20",
50-
"156.154.70.22",
51-
"77.88.8.7",
52-
"77.88.8.8",
53-
"::1",
54-
"2a02:6b8::feed:0ff",
55-
"2a02:6b8::feed:bad",
56-
"2a02:6b8::feed:a11",
57-
"2620:119:35::35",
58-
"2620:119:53::53",
59-
"2606:4700:4700::1111",
60-
"2606:4700:4700::1001",
61-
"2001:4860:4860::8888",
62-
"2a0d:2a00:1::",
63-
"2a0d:2a00:2::",
64-
}
65-
66-
type Blocklist struct {
67-
Server string `json:"server"`
68-
ServerIP string `json:"serverIp"`
69-
IsBlocked bool `json:"isBlocked"`
70-
}
71-
72-
func isDomainBlocked(domain, serverIP string) bool {
73-
resolver := &net.Resolver{
74-
PreferGo: true,
75-
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
76-
d := net.Dialer{
77-
Timeout: time.Second * 3,
78-
}
79-
return d.DialContext(ctx, network, serverIP+":53")
80-
},
81-
}
82-
83-
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
84-
defer cancel()
855

86-
ips, err := resolver.LookupIP(ctx, "ip4", domain)
87-
if err != nil {
88-
// if there's an error, consider it not blocked
89-
return false
90-
}
91-
92-
return slices.ContainsFunc(ips, func(ip net.IP) bool {
93-
return slices.Contains(knownBlockIPs, ip.String())
94-
})
95-
}
96-
97-
func checkDomainAgainstDNSServers(domain string) []Blocklist {
98-
var lock sync.Mutex
99-
var wg sync.WaitGroup
100-
limit := make(chan struct{}, 5)
101-
102-
var results []Blocklist
103-
104-
for _, server := range DNS_SERVERS {
105-
wg.Add(1)
106-
go func(server dnsServer) {
107-
limit <- struct{}{}
108-
defer func() {
109-
<-limit
110-
wg.Done()
111-
}()
112-
113-
isBlocked := isDomainBlocked(domain, server.IP)
114-
lock.Lock()
115-
defer lock.Unlock()
116-
results = append(results, Blocklist{
117-
Server: server.Name,
118-
ServerIP: server.IP,
119-
IsBlocked: isBlocked,
120-
})
121-
}(server)
122-
}
123-
wg.Wait()
124-
125-
sort.Slice(results, func(i, j int) bool {
126-
return results[i].Server > results[j].Server
127-
})
128-
return results
129-
}
6+
"github.com/xray-web/web-check-api/checks"
7+
)
1308

131-
func HandleBlockLists() http.Handler {
132-
type Response struct {
133-
BlockLists []Blocklist `json:"blocklists"`
134-
}
9+
func HandleBlockLists(b *checks.BlockList) http.Handler {
13510
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
13611
rawURL, err := extractURL(r)
13712
if err != nil {
13813
JSONError(w, ErrMissingURLParameter, http.StatusBadRequest)
13914
return
14015
}
141-
json.NewEncoder(w).Encode(Response{BlockLists: checkDomainAgainstDNSServers(rawURL.Hostname())})
16+
list := b.BlockedServers(r.Context(), rawURL.Hostname())
17+
JSON(w, list, http.StatusOK)
14218
})
14319
}

0 commit comments

Comments
 (0)