Merge pull request #1611 from tdyas/digitalocean_increase_per_page

digitalocean: increase API page size

docs/tutorials/digitalocean.md

@@ -189,3 +189,13 @@ Now that we have verified that ExternalDNS will automatically manage DigitalOcea
 $ kubectl delete service -f nginx.yaml
 $ kubectl delete service -f externaldns.yaml
 ```
+
+## Advanced Usage
+
+### API Page Size
+
+If you have a large number of domains and/or records within a domain, you may encounter API
+rate limiting because of the number of API calls that external-dns must make to the DigitalOcean API to retrieve
+the current DNS configuration during every reconciliation loop. If this is the case, use the 
+`--digitalocean-api-page-size` option to increase the size of the pages used when querying the DigitalOcean API.
+(Note: external-dns uses a default of 50.)

main.go

@@ -198,7 +198,7 @@ func main() {
 	case "google":
 		p, err = google.NewGoogleProvider(ctx, cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.DryRun)
 	case "digitalocean":
-		p, err = digitalocean.NewDigitalOceanProvider(ctx, domainFilter, cfg.DryRun)
+		p, err = digitalocean.NewDigitalOceanProvider(ctx, domainFilter, cfg.DryRun, cfg.DigitalOceanAPIPageSize)
 	case "ovh":
 		p, err = ovh.NewOVHProvider(ctx, domainFilter, cfg.OVHEndpoint, cfg.DryRun)
 	case "linode":

pkg/apis/externaldns/types.go

@@ -140,6 +140,7 @@ type Config struct {
 	NS1IgnoreSSL                      bool
 	TransIPAccountName                string
 	TransIPPrivateKeyFile             string
+	DigitalOceanAPIPageSize           int
 }
 
 var defaultConfig = &Config{
@@ -237,6 +238,7 @@ var defaultConfig = &Config{
 	NS1IgnoreSSL:                false,
 	TransIPAccountName:          "",
 	TransIPPrivateKeyFile:       "",
+	DigitalOceanAPIPageSize:     50,
 }
 
 // NewConfig returns new Config object
@@ -363,6 +365,7 @@ func (cfg *Config) ParseFlags(args []string) error {
 	app.Flag("pdns-tls-enabled", "When using the PowerDNS/PDNS provider, specify whether to use TLS (default: false, requires --tls-ca, optionally specify --tls-client-cert and --tls-client-cert-key)").Default(strconv.FormatBool(defaultConfig.PDNSTLSEnabled)).BoolVar(&cfg.PDNSTLSEnabled)
 	app.Flag("ns1-endpoint", "When using the NS1 provider, specify the URL of the API endpoint to target (default: https://api.nsone.net/v1/)").Default(defaultConfig.NS1Endpoint).StringVar(&cfg.NS1Endpoint)
 	app.Flag("ns1-ignoressl", "When using the NS1 provider, specify whether to verify the SSL certificate (default: false)").Default(strconv.FormatBool(defaultConfig.NS1IgnoreSSL)).BoolVar(&cfg.NS1IgnoreSSL)
+	app.Flag("digitalocean-api-page-size", "Configure the page size used when querying the DigitalOcean API.").Default(strconv.Itoa(defaultConfig.DigitalOceanAPIPageSize)).IntVar(&cfg.DigitalOceanAPIPageSize)
 
 	// Flags related to TLS communication
 	app.Flag("tls-ca", "When using TLS communication, the path to the certificate authority to verify server communications (optionally specify --tls-client-cert for two-way TLS)").Default(defaultConfig.TLSCA).StringVar(&cfg.TLSCA)

pkg/apis/externaldns/types_test.go

@@ -98,6 +98,7 @@ var (
 		RcodezeroTXTEncrypt:         false,
 		TransIPAccountName:          "",
 		TransIPPrivateKeyFile:       "",
+		DigitalOceanAPIPageSize:     50,
 	}
 
 	overriddenConfig = &Config{
@@ -177,6 +178,7 @@ var (
 		NS1IgnoreSSL:                true,
 		TransIPAccountName:          "transip",
 		TransIPPrivateKeyFile:       "/path/to/transip.key",
+		DigitalOceanAPIPageSize:     100,
 	}
 )
 
@@ -280,6 +282,7 @@ func TestParseFlags(t *testing.T) {
 				"--ns1-ignoressl",
 				"--transip-account=transip",
 				"--transip-keyfile=/path/to/transip.key",
+				"--digitalocean-api-page-size=100",
 			},
 			envVars:  map[string]string{},
 			expected: overriddenConfig,
@@ -364,6 +367,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_NS1_IGNORESSL":                   "1",
 				"EXTERNAL_DNS_TRANSIP_ACCOUNT":                 "transip",
 				"EXTERNAL_DNS_TRANSIP_KEYFILE":                 "/path/to/transip.key",
+				"EXTERNAL_DNS_DIGITALOCEAN_API_PAGE_SIZE":      "100",
 			},
 			expected: overriddenConfig,
 		},

provider/digitalocean/digital_ocean.go

@@ -49,7 +49,9 @@ type DigitalOceanProvider struct {
 	Client godo.DomainsService
 	// only consider hosted zones managing domains ending in this suffix
 	domainFilter endpoint.DomainFilter
-	DryRun       bool
+	// page size when querying paginated APIs
+	apiPageSize int
+	DryRun      bool
 }
 
 // DigitalOceanChange differentiates between ChangActions
@@ -59,7 +61,7 @@ type DigitalOceanChange struct {
 }
 
 // NewDigitalOceanProvider initializes a new DigitalOcean DNS based Provider.
-func NewDigitalOceanProvider(ctx context.Context, domainFilter endpoint.DomainFilter, dryRun bool) (*DigitalOceanProvider, error) {
+func NewDigitalOceanProvider(ctx context.Context, domainFilter endpoint.DomainFilter, dryRun bool, apiPageSize int) (*DigitalOceanProvider, error) {
 	token, ok := os.LookupEnv("DO_TOKEN")
 	if !ok {
 		return nil, fmt.Errorf("no token found")
@@ -72,6 +74,7 @@ func NewDigitalOceanProvider(ctx context.Context, domainFilter endpoint.DomainFi
 	provider := &DigitalOceanProvider{
 		Client:       client.Domains,
 		domainFilter: domainFilter,
+		apiPageSize:  apiPageSize,
 		DryRun:       dryRun,
 	}
 	return provider, nil
@@ -128,7 +131,7 @@ func (p *DigitalOceanProvider) Records(ctx context.Context) ([]*endpoint.Endpoin
 
 func (p *DigitalOceanProvider) fetchRecords(ctx context.Context, zoneName string) ([]godo.DomainRecord, error) {
 	allRecords := []godo.DomainRecord{}
-	listOptions := &godo.ListOptions{}
+	listOptions := &godo.ListOptions{PerPage: p.apiPageSize}
 	for {
 		records, resp, err := p.Client.Records(ctx, zoneName, listOptions)
 		if err != nil {
@@ -153,7 +156,7 @@ func (p *DigitalOceanProvider) fetchRecords(ctx context.Context, zoneName string
 
 func (p *DigitalOceanProvider) fetchZones(ctx context.Context) ([]godo.Domain, error) {
 	allZones := []godo.Domain{}
-	listOptions := &godo.ListOptions{}
+	listOptions := &godo.ListOptions{PerPage: p.apiPageSize}
 	for {
 		zones, resp, err := p.Client.List(ctx, listOptions)
 		if err != nil {

provider/digitalocean/digital_ocean_test.go

@@ -187,12 +187,12 @@ func TestDigitalOceanApplyChanges(t *testing.T) {
 
 func TestNewDigitalOceanProvider(t *testing.T) {
 	_ = os.Setenv("DO_TOKEN", "xxxxxxxxxxxxxxxxx")
-	_, err := NewDigitalOceanProvider(context.Background(), endpoint.NewDomainFilter([]string{"ext-dns-test.zalando.to."}), true)
+	_, err := NewDigitalOceanProvider(context.Background(), endpoint.NewDomainFilter([]string{"ext-dns-test.zalando.to."}), true, 50)
 	if err != nil {
 		t.Errorf("should not fail, %s", err)
 	}
 	_ = os.Unsetenv("DO_TOKEN")
-	_, err = NewDigitalOceanProvider(context.Background(), endpoint.NewDomainFilter([]string{"ext-dns-test.zalando.to."}), true)
+	_, err = NewDigitalOceanProvider(context.Background(), endpoint.NewDomainFilter([]string{"ext-dns-test.zalando.to."}), true, 50)
 	if err == nil {
 		t.Errorf("expected to fail")
 	}