Kharec/goyco
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: SSRF DNS rebinding

Browse Source
This commit is contained in:
Kharec
2026-07-23 16:41:17 +02:00
parent 96958af310
commit e2c46c9e58
2 changed files with 190 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/services/url_metadata_service.go
+99 -164
View File
@@ -7,6 +7,7 @@ import (
"io"
"net"
"net/http"
"net/netip"
"net/url"
"slices"
"strings"
@@ -32,6 +33,9 @@ const (
tlsHandshakeTimeout = 5 * time.Second
responseHeaderTimeout = 5 * time.Second
maxContentLength = 10 * 1024 * 1024
dnsPinTTL = 5 * time.Minute
maxDNSPinnedHosts = 1024
)
type TitleFetcher interface {
@@ -48,65 +52,66 @@ func (d DefaultDNSResolver) LookupIP(hostname string) ([]net.IP, error) {
return net.LookupIP(hostname)
}
type dnsCacheEntry struct {
ips []net.IP
expiresAt time.Time
}
type DNSCache struct {
mu sync.RWMutex
data map[string][]net.IP
data map[string]dnsCacheEntry
}
func NewDNSCache() *DNSCache {
return &DNSCache{
data: make(map[string][]net.IP),
data: make(map[string]dnsCacheEntry),
}
}
func (c *DNSCache) Get(hostname string) ([]net.IP, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
ips, exists := c.data[hostname]
return ips, exists
entry, exists := c.data[hostname]
if !exists || time.Now().After(entry.expiresAt) {
return nil, false
}
return entry.ips, true
}
func (c *DNSCache) Set(hostname string, ips []net.IP) {
c.mu.Lock()
defer c.mu.Unlock()
c.data[hostname] = ips
}
type CachedDNSResolver struct {
resolver DNSResolver
cache *DNSCache
}
func NewCachedDNSResolver(resolver DNSResolver) *CachedDNSResolver {
return &CachedDNSResolver{
resolver: resolver,
cache: NewDNSCache(),
}
}
func (c *CachedDNSResolver) LookupIP(hostname string) ([]net.IP, error) {
if ips, exists := c.cache.Get(hostname); exists {
return ips, nil
if len(c.data) >= maxDNSPinnedHosts {
now := time.Now()
for host, entry := range c.data {
if now.After(entry.expiresAt) {
delete(c.data, host)
}
}
for host := range c.data {
if len(c.data) < maxDNSPinnedHosts {
break
}
delete(c.data, host)
}
}
ips, err := c.resolver.LookupIP(hostname)
if err != nil {
return nil, err
c.data[hostname] = dnsCacheEntry{
ips: ips,
expiresAt: time.Now().Add(dnsPinTTL),
}
c.cache.Set(hostname, ips)
return ips, nil
}
type CustomDialer struct {
cache *DNSCache
fallback *net.Dialer
cache *DNSCache
dialer *net.Dialer
}
func NewCustomDialer(cache *DNSCache) *CustomDialer {
return &CustomDialer{
cache: cache,
fallback: &net.Dialer{
dialer: &net.Dialer{
Timeout: dialTimeout,
},
}
@@ -115,40 +120,43 @@ func NewCustomDialer(cache *DNSCache) *CustomDialer {
func (d *CustomDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return d.fallback.DialContext(ctx, network, address)
return nil, ErrSSRFBlocked
}
if ips, exists := d.cache.Get(host); exists {
for _, ip := range ips {
ipAddr := net.JoinHostPort(ip.String(), port)
if conn, err := d.fallback.DialContext(ctx, network, ipAddr); err == nil {
return conn, nil
}
ips, exists := d.cache.Get(host)
if !exists {
return nil, ErrSSRFBlocked
}
var lastErr error
for _, ip := range ips {
conn, err := d.dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
if err == nil {
return conn, nil
}
lastErr = err
}
return d.fallback.DialContext(ctx, network, address)
if lastErr == nil {
lastErr = ErrSSRFBlocked
}
return nil, lastErr
}
type URLMetadataService struct {
client *http.Client
resolver DNSResolver
dnsCache *DNSCache
approvedHosts map[string]bool
mu sync.RWMutex
client *http.Client
resolver DNSResolver
dnsCache *DNSCache
}
func NewURLMetadataService() *URLMetadataService {
dnsCache := NewDNSCache()
cachedResolver := NewCachedDNSResolver(DefaultDNSResolver{})
customDialer := NewCustomDialer(dnsCache)
svc := &URLMetadataService{
resolver: cachedResolver,
dnsCache: dnsCache,
approvedHosts: make(map[string]bool),
resolver: DefaultDNSResolver{},
dnsCache: NewDNSCache(),
}
customDialer := NewCustomDialer(svc.dnsCache)
transport := &http.Transport{
DialContext: customDialer.DialContext,
MaxIdleConns: 100,
@@ -166,25 +174,7 @@ func NewURLMetadataService() *URLMetadataService {
if len(via) >= maxRedirects {
return ErrTooManyRedirects
}
hostname := req.URL.Hostname()
svc.mu.RLock()
approved := svc.approvedHosts[hostname]
svc.mu.RUnlock()
if approved {
return nil
}
if err := svc.validateURLForSSRF(req.URL); err != nil {
return err
}
svc.mu.Lock()
svc.approvedHosts[hostname] = true
svc.mu.Unlock()
return nil
return svc.validateURLForSSRF(req.URL)
},
}
return svc
@@ -204,19 +194,8 @@ func (s *URLMetadataService) FetchTitle(ctx context.Context, rawURL string) (str
return "", ErrUnsupportedScheme
}
hostname := parsed.Hostname()
s.mu.RLock()
approved := s.approvedHosts[hostname]
s.mu.RUnlock()
if !approved {
if err := s.validateURLForSSRF(parsed); err != nil {
return "", err
}
s.mu.Lock()
s.approvedHosts[hostname] = true
s.mu.Unlock()
if err := s.validateURLForSSRF(parsed); err != nil {
return "", err
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
@@ -336,13 +315,20 @@ func (s *URLMetadataService) validateURLForSSRF(u *url.URL) error {
return ErrSSRFBlocked
}
ips, err := s.resolver.LookupIP(u.Hostname())
if err != nil {
hostname := u.Hostname()
if _, pinned := s.dnsCache.Get(hostname); pinned {
return nil
}
ips, err := s.resolver.LookupIP(hostname)
if err != nil || len(ips) == 0 {
return ErrSSRFBlocked
}
if slices.ContainsFunc(ips, isPrivateOrReservedIP) {
return ErrSSRFBlocked
}
s.dnsCache.Set(hostname, ips)
return nil
}
@@ -361,93 +347,42 @@ func isLocalhost(hostname string) bool {
return slices.Contains(localhostNames, hostname)
}
var reservedPrefixes = []netip.Prefix{
netip.MustParsePrefix("0.0.0.0/8"),
netip.MustParsePrefix("100.64.0.0/10"),
netip.MustParsePrefix("192.0.0.0/24"),
netip.MustParsePrefix("192.0.2.0/24"),
netip.MustParsePrefix("198.18.0.0/15"),
netip.MustParsePrefix("198.51.100.0/24"),
netip.MustParsePrefix("203.0.113.0/24"),
netip.MustParsePrefix("240.0.0.0/4"),
netip.MustParsePrefix("64:ff9b::/96"),
netip.MustParsePrefix("100::/64"),
netip.MustParsePrefix("2001:db8::/32"),
}
func isPrivateOrReservedIP(ip net.IP) bool {
if ip == nil {
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return true
}
addr = addr.Unmap()
if addr.IsLoopback() ||
addr.IsPrivate() ||
addr.IsLinkLocalUnicast() ||
addr.IsLinkLocalMulticast() ||
addr.IsInterfaceLocalMulticast() ||
addr.IsMulticast() ||
addr.IsUnspecified() {
return true
}
ipv4 := ip.To4()
if ipv4 == nil {
return isPrivateIPv6(ip)
}
privateRanges := []struct {
start, end net.IP
}{
{net.IPv4(10, 0, 0, 0), net.IPv4(10, 255, 255, 255)},
{net.IPv4(172, 16, 0, 0), net.IPv4(172, 31, 255, 255)},
{net.IPv4(192, 168, 0, 0), net.IPv4(192, 168, 255, 255)},
{net.IPv4(127, 0, 0, 0), net.IPv4(127, 255, 255, 255)},
{net.IPv4(169, 254, 0, 0), net.IPv4(169, 254, 255, 255)},
{net.IPv4(224, 0, 0, 0), net.IPv4(239, 255, 255, 255)},
{net.IPv4(240, 0, 0, 0), net.IPv4(255, 255, 255, 255)},
}
for _, r := range privateRanges {
if ipInRange(ipv4, r.start, r.end) {
for _, prefix := range reservedPrefixes {
if prefix.Contains(addr) {
return true
}
}
return false
}
func isPrivateIPv6(ip net.IP) bool {
privateRanges := []struct {
prefix []byte
length int
}{
{[]byte{0xfc, 0x00}, 7},
{[]byte{0xfe, 0x80}, 10},
{[]byte{0xff, 0x00}, 8},
{[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, 128},
}
for _, r := range privateRanges {
if ipv6InRange(ip, r.prefix, r.length) {
return true
}
}
return false
}
func ipInRange(ip, start, end net.IP) bool {
ipInt := ipToInt(ip)
startInt := ipToInt(start)
endInt := ipToInt(end)
return ipInt >= startInt && ipInt <= endInt
}
func ipToInt(ip net.IP) uint32 {
ipv4 := ip.To4()
if ipv4 == nil {
return 0
}
return uint32(ipv4[0])<<24 + uint32(ipv4[1])<<16 + uint32(ipv4[2])<<8 + uint32(ipv4[3])
}
func ipv6InRange(ip net.IP, prefix []byte, length int) bool {
ipBytes := ip.To16()
if ipBytes == nil {
return false
}
bytesToCompare := length / 8
bitsToCompare := length % 8
for i := 0; i < bytesToCompare && i < len(prefix) && i < len(ipBytes); i++ {
if ipBytes[i] != prefix[i] {
return false
}
}
if bitsToCompare > 0 && bytesToCompare < len(prefix) && bytesToCompare < len(ipBytes) {
mask := byte(0xff) << (8 - bitsToCompare)
if (ipBytes[bytesToCompare] & mask) != (prefix[bytesToCompare] & mask) {
return false
}
}
return true
}
internal/services/url_metadata_service_test.go
+91 -74
View File
@@ -9,6 +9,7 @@ import (
"net/url"
"strings"
"testing"
"time"
)
func TestFetchTitleSuccess(t *testing.T) {
@@ -428,10 +429,38 @@ func TestIsPrivateOrReservedIP(t *testing.T) {
{"169.254.0.1", "169.254.0.1", true},
{"224.0.0.1", "224.0.0.1", true},
{"240.0.0.1", "240.0.0.1", true},
{"0.0.0.0", "0.0.0.0", true},
{"0.1.2.3", "0.1.2.3", true},
{"CGNAT 100.64.0.1", "100.64.0.1", true},
{"CGNAT 100.127.255.255", "100.127.255.255", true},
{"IETF 192.0.0.1", "192.0.0.1", true},
{"TEST-NET-1 192.0.2.1", "192.0.2.1", true},
{"benchmarking 198.18.0.1", "198.18.0.1", true},
{"benchmarking 198.19.255.255", "198.19.255.255", true},
{"TEST-NET-2 198.51.100.1", "198.51.100.1", true},
{"TEST-NET-3 203.0.113.1", "203.0.113.1", true},
{"broadcast 255.255.255.255", "255.255.255.255", true},
{"IPv4-mapped private ::ffff:10.0.0.1", "::ffff:10.0.0.1", true},
{"IPv4-mapped loopback ::ffff:127.0.0.1", "::ffff:127.0.0.1", true},
{"IPv6 loopback ::1", "::1", true},
{"IPv6 unspecified ::", "::", true},
{"IPv6 ULA fc00::1", "fc00::1", true},
{"IPv6 ULA fd00::1", "fd00::1", true},
{"IPv6 link-local fe80::1", "fe80::1", true},
{"IPv6 multicast ff00::1", "ff00::1", true},
{"IPv6 interface-local multicast ff01::1", "ff01::1", true},
{"NAT64 64:ff9b::7f00:1", "64:ff9b::7f00:1", true},
{"discard-only 100::1", "100::1", true},
{"documentation 2001:db8::1", "2001:db8::1", true},
{"8.8.8.8", "8.8.8.8", false},
{"1.1.1.1", "1.1.1.1", false},
{"74.125.224.72", "74.125.224.72", false},
{"100.128.0.1 just past CGNAT", "100.128.0.1", false},
{"IPv6 public 2001:4860::1", "2001:4860::1", false},
{"IPv6 public 2607:f8b0::1", "2607:f8b0::1", false},
{"IPv4-mapped public ::ffff:8.8.8.8", "::ffff:8.8.8.8", false},
{"nil IP", "", true},
}
@@ -848,93 +877,81 @@ func (c *CountingMockDNSResolver) LookupIP(hostname string) ([]net.IP, error) {
return c.MockDNSResolver.LookupIP(hostname)
}
func TestIPv6PrivateRangeDetection(t *testing.T) {
tests := []struct {
name string
ip string
expected bool
}{
{"fc00::1", "fc00::1", true},
{"fe80::1", "fe80::1", true},
{"ff00::1", "ff00::1", true},
{"::1", "::1", true},
{"2001:db8::1", "2001:db8::1", false},
{"2001:4860::1", "2001:4860::1", false},
{"2607:f8b0::1", "2607:f8b0::1", false},
{"invalid", "invalid", false},
{"", "", false},
func TestCustomDialerRequiresPinnedIP(t *testing.T) {
cache := NewDNSCache()
dialer := NewCustomDialer(cache)
_, err := dialer.DialContext(context.Background(), "tcp", "unvalidated.example.com:443")
if !errors.Is(err, ErrSSRFBlocked) {
t.Fatalf("expected ErrSSRFBlocked for unpinned host, got %v", err)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var ip net.IP
if tt.ip != "" && tt.ip != "invalid" {
ip = net.ParseIP(tt.ip)
}
result := isPrivateIPv6(ip)
if result != tt.expected {
t.Fatalf("expected %v for IPv6 %q, got %v", tt.expected, tt.ip, result)
}
})
_, err = dialer.DialContext(context.Background(), "tcp", "missing-port.example.com")
if !errors.Is(err, ErrSSRFBlocked) {
t.Fatalf("expected ErrSSRFBlocked for invalid address, got %v", err)
}
}
func TestIPRangeDetection(t *testing.T) {
tests := []struct {
name string
ip string
start string
end string
expected bool
}{
{"IP in range", "192.168.1.100", "192.168.1.1", "192.168.1.255", true},
{"IP at start of range", "192.168.1.1", "192.168.1.1", "192.168.1.255", true},
{"IP at end of range", "192.168.1.255", "192.168.1.1", "192.168.1.255", true},
{"IP below range", "192.168.0.255", "192.168.1.1", "192.168.1.255", false},
{"IP above range", "192.168.2.1", "192.168.1.1", "192.168.1.255", false},
{"Same IP", "192.168.1.100", "192.168.1.100", "192.168.1.100", true},
func TestDNSCacheExpiry(t *testing.T) {
cache := NewDNSCache()
cache.Set("example.com", []net.IP{net.ParseIP("8.8.8.8")})
if _, exists := cache.Get("example.com"); !exists {
t.Fatal("expected fresh entry to be returned")
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ip := net.ParseIP(tt.ip)
start := net.ParseIP(tt.start)
end := net.ParseIP(tt.end)
cache.mu.Lock()
entry := cache.data["example.com"]
entry.expiresAt = time.Now().Add(-time.Second)
cache.data["example.com"] = entry
cache.mu.Unlock()
result := ipInRange(ip, start, end)
if result != tt.expected {
t.Fatalf("expected %v for IP %q in range %q-%q, got %v", tt.expected, tt.ip, tt.start, tt.end, result)
}
})
if _, exists := cache.Get("example.com"); exists {
t.Fatal("expected expired entry to be treated as a miss")
}
}
func TestIPv6RangeDetection(t *testing.T) {
tests := []struct {
name string
ip string
prefix []byte
length int
expected bool
}{
{"fc00 prefix match", "fc00::1", []byte{0xfc, 0x00}, 7, true},
{"fc00 prefix no match", "fd00::1", []byte{0xfc, 0x00}, 7, true},
{"fe80 prefix match", "fe80::1", []byte{0xfe, 0x80}, 10, true},
{"fe80 prefix no match", "fe90::1", []byte{0xfe, 0x80}, 10, true},
{"ff00 prefix match", "ff00::1", []byte{0xff, 0x00}, 8, true},
{"ff00 prefix no match", "fe00::1", []byte{0xff, 0x00}, 8, false},
{"exact match", "::1", []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, 128, true},
func TestExpiredPinTriggersRevalidation(t *testing.T) {
svc := NewURLMetadataService()
lookupCount := 0
mockResolver := &CountingMockDNSResolver{
MockDNSResolver: MockDNSResolver{
lookupResults: make(map[string][]net.IP),
lookupErrors: make(map[string]error),
},
lookupCount: &lookupCount,
}
mockResolver.SetLookupResult("example.com", []net.IP{net.ParseIP("8.8.8.8")})
svc.resolver = mockResolver
svc.client = newTestClient(t, func(r *http.Request) (*http.Response, error) {
body := io.NopCloser(strings.NewReader("<html><head><title>Test Title</title></head></html>"))
header := make(http.Header)
header.Set("Content-Type", "text/html; charset=utf-8")
return &http.Response{StatusCode: http.StatusOK, Body: body, Header: header}, nil
})
if _, err := svc.FetchTitle(context.Background(), "https://example.com"); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if lookupCount != 1 {
t.Fatalf("expected 1 DNS lookup, got %d", lookupCount)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ip := net.ParseIP(tt.ip)
result := ipv6InRange(ip, tt.prefix, tt.length)
if result != tt.expected {
t.Fatalf("expected %v for IPv6 %q with prefix %v/%d, got %v", tt.expected, tt.ip, tt.prefix, tt.length, result)
}
})
svc.dnsCache.mu.Lock()
entry := svc.dnsCache.data["example.com"]
entry.expiresAt = time.Now().Add(-time.Second)
svc.dnsCache.data["example.com"] = entry
svc.dnsCache.mu.Unlock()
mockResolver.SetLookupResult("example.com", []net.IP{net.ParseIP("127.0.0.1")})
if _, err := svc.FetchTitle(context.Background(), "https://example.com"); !errors.Is(err, ErrSSRFBlocked) {
t.Fatalf("expected ErrSSRFBlocked after DNS rebind to private IP, got %v", err)
}
if lookupCount != 2 {
t.Fatalf("expected expired pin to trigger a second DNS lookup, got %d", lookupCount)
}
}