Skip to content

Instantly share code, notes, and snippets.

@Integralist

Integralist/README.md

Last active July 7, 2025 15:07
Show Gist options
  • Save Integralist/8a9cb8924f75ae42487fd877b03360e2 to your computer and use it in GitHub Desktop.
Save Integralist/8a9cb8924f75ae42487fd877b03360e2 to your computer and use it in GitHub Desktop.
Download ZIP
Go: timeouts and custom http client #go #http #dns
Raw
README.md

Go Network Timeouts

NOTE: Guide to net/http timeouts

Although not explicitly stated, DNS resolution appears to be taken into consideration as part of the overall http.Client.Timeout setting. If you need to set your own DNS timeout, then it seems https://github.com/miekg/dns is a popular solution.

Raw
basic go http client configuration.go
package main
import (
"crypto/tls"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"syscall"
"time"
)
func main() {
client := &http.Client{
Timeout: time.Second * time.Duration(5*time.Second),
Transport: &http.Transport{
// Avoid: "x509: certificate signed by unknown authority"
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
// Inspect the network connection type
DialContext: (&net.Dialer{
Control: func(network, address string, c syscall.RawConn) error {
// Reference: https://golang.org/pkg/net/#Dial
if network == "tcp4" {
return errors.New("we don't want you to use IPv4")
}
return nil
},
}).DialContext,
},
}
req, err := http.NewRequest("GET", "https://ipv4.lookup.test-ipv6.com/", nil)
if err != nil {
log.Fatal(err)
}
res, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
b, err := io.ReadAll(res.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%+v\n", string(b))
}
Raw
custom dns resolver.go
package main
import (
"context"
"io/ioutil"
"log"
"net"
"net/http"
"time"
)
func main() {
var (
dnsResolverIP = "8.8.8.8:53" // Google DNS resolver.
dnsResolverProto = "udp" // Protocol to use for the DNS resolver
dnsResolverTimeoutMs = 5000 // Timeout (ms) for the DNS resolver (optional)
)
dialer := &net.Dialer{
Resolver: &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
d := net.Dialer{
Timeout: time.Duration(dnsResolverTimeoutMs) * time.Millisecond,
}
return d.DialContext(ctx, dnsResolverProto, dnsResolverIP)
},
},
}
dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
return dialer.DialContext(ctx, network, addr)
}
http.DefaultTransport.(*http.Transport).DialContext = dialContext
httpClient := &http.Client{}
// Testing the new HTTP client with the custom DNS resolver.
resp, err := httpClient.Get("https://www.google.com")
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
log.Println(string(body))
}
Raw
force IPv4 connection type (standard library implementation).go
package main
import (
"context"
"crypto/tls"
"fmt"
"io"
"log"
"net"
"net/http"
"time"
)
func main() {
client := &http.Client{
Timeout: time.Second * time.Duration(5*time.Second),
Transport: &http.Transport{
// Avoid: "x509: certificate signed by unknown authority"
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "tcp4", addr)
},
},
}
// Fastly's DNS system controls whether we will report IPv6 addresses for a
// given hostname, and in the case of developer.fastly.com it CNAMEs to the
// Fastly map devhub.fastly.net which is configured to opt-in or out of v6
// support at the map level. The devhub map has dual-stack enabled on it.
// Therefore, it will announce v6 addresses for it if a client sends AAAA DNS
// queries for the hostname.
req, err := http.NewRequest("GET", "https://developer.fastly.com/api/internal/cli-config", nil)
if err != nil {
log.Fatal(err)
}
res, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
b, err := io.ReadAll(res.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%+v\n", string(b))
}
Raw
force IPv4 connection type (uses third-party dependency).go
package main
import (
"context"
"crypto/tls"
"fmt"
"io"
"log"
"net"
"net/http"
"strings"
"time"
"github.com/miekg/dns"
)
func main() {
client := &http.Client{
Timeout: time.Second * time.Duration(5*time.Second),
Transport: &http.Transport{
// Avoid: "x509: certificate signed by unknown authority"
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
ipv4, err := resolveIPv4(addr)
if err != nil {
return nil, err
}
timeout, err := time.ParseDuration("10s")
if err != nil {
return nil, err
}
return (&net.Dialer{
Timeout: timeout,
}).DialContext(ctx, network, ipv4)
},
},
}
// Also try: https://v4.testmyipv6.com/
req, err := http.NewRequest("GET", "https://ipv4.lookup.test-ipv6.com/", nil)
if err != nil {
log.Fatal(err)
}
res, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
b, err := io.ReadAll(res.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%+v\n", string(b))
}
// resolveIPv4 resolves an address to IPv4 address.
func resolveIPv4(addr string) (string, error) {
url := strings.Split(addr, ":")
m := new(dns.Msg)
m.SetQuestion(dns.Fqdn(url[0]), dns.TypeA)
m.RecursionDesired = true
// NOTE: you shouldn't consult or rely on /etc/resolv.conf as it has proven historically to contain nameservers that don't respond.
config, _ := dns.ClientConfigFromFile("/etc/resolv.conf")
c := new(dns.Client)
r, _, err := c.Exchange(m, net.JoinHostPort(config.Servers[0], config.Port))
if err != nil {
return "", err
}
for _, ans := range r.Answer {
if a, ok := ans.(*dns.A); ok {
url[0] = a.A.String()
}
}
return strings.Join(url, ":"), nil
}
Raw
skip DNS resolution and just pass an IP.go
// This enables you to utilise a package such as https://github.com/miekg/dns to resolve the hostname.
package main
import (
"context"
"io/ioutil"
"log"
"net"
"net/http"
"time"
)
func main() {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
http.DefaultTransport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
if addr == "google.com:443" {
addr = "216.58.198.206:443"
}
return dialer.DialContext(ctx, network, addr)
}
resp, err := http.Get("https://www.google.com")
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
log.Println(string(body))
}
@thepabloaguilar
Copy link

Hey @Integralist, great gist! In some cases we can have more than one DNS server, I'd like to know if in that case the code below is right (multiples DNS servers forcing IPV4 resolution):

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/miekg/dns"
)

func main() {
	client := &http.Client{
		Timeout: time.Second * 5,
		Transport: &http.Transport{
			// Avoid: "x509: certificate signed by unknown authority"
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
			DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
				ipv4, err := resolveIPv4(addr)
				if err != nil {
					return nil, err
				}
				timeout, err := time.ParseDuration("10s")
				if err != nil {
					return nil, err
				}
				return (&net.Dialer{
					Timeout: timeout,
				}).DialContext(ctx, network, ipv4)
			},
		},
	}

	// Also try: https://v4.testmyipv6.com/
	req, err := http.NewRequest("GET", "https://ipv4.lookup.test-ipv6.com/", nil)
	if err != nil {
		log.Fatal(err)
	}

	res, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	b, err := io.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%+v\n", string(b))
}

// resolveIPv4 resolves an address to IPv4 address.
func resolveIPv4(addr string) (string, error) {
	url := strings.Split(addr, ":")

	m := &dns.Msg{
		MsgHdr: dns.MsgHdr{
			RecursionDesired: true,
		},
	}
	m.SetQuestion(dns.Fqdn(url[0]), dns.TypeA)

	// NOTE: you shouldn't consult or rely on /etc/resolv.conf as it has proven historically to contain nameservers that don't respond.
	config, _ := dns.ClientConfigFromFile("/etc/resolv.conf")
	c := new(dns.Client)

	var err error
	for _, server := range config.Servers {
		r, _, innerErr := c.Exchange(m, net.JoinHostPort(server, config.Port))
		if innerErr != nil {
			err = innerErr
			continue
		}

		for _, ans := range r.Answer {
			if a, ok := ans.(*dns.A); ok {
				url[0] = a.A.String()
			}
		}

		return strings.Join(url, ":"), nil
	}

	return "", err
}

@Integralist
Copy link
Author

@thepabloaguilar I think you might want to append errors to your var err error rather than reset it on each loop iteration...

if innerErr != nil {
	err = fmt.Errorf("%w: %w", err, innerErr)
	continue
}

Otherwise, multiple servers might fail to resolve and you'd only know about the last one.

Also, one thing I discovered recently was the issue of truncation. See https://pkg.go.dev/github.com/miekg/dns#Client.Exchange

Exchange does not retry a failed query, nor will it fall back to TCP in case of truncation. It is up to the caller to create a message that allows for larger responses to be returned. Specifically this means adding an EDNS0 OPT RR that will advertise a larger buffer, see SetEdns0.

Check the implementation in this PR for an example:
domainr/dnsr#118

@thepabloaguilar
Copy link

Yeah, appending makes sense but probably I'm gonna put some OTel spans inside it so the last error will be enough!

Talking about the truncation issue, I could just create another "dns.Client" setting the "Net" attribute to "tcp", right?

@thepabloaguilar
Copy link

I'm thinking on leave the control of all the DNS calls to Golang by providing that "Resolver.Dial" function, everytime the function is called the code get the next DNS address. This approach has two great things (at least for my use case):

  • If an error occurs, Go will retry and the next retry will be a different DNS address
  • Round-Robin the DNS servers

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment