Commit dddc4d52 authored by Andrei Mihu's avatar Andrei Mihu
Browse files

New configuration option for frequency of database DNS change scans.

parent 88c33a22

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ The format is based on [keep a changelog](http://keepachangelog.com) and this pr 
### Added

- Add ctx field to access http request headers in the runtimes.

- New JS runtime stringToBinary and binaryToString functions.

- New configuration option for frequency of database DNS change scans.



### Fixed

- Gracefully close Lua matches when call queue fills up.

server/config.go

+13 −8
Original line number Diff line number Diff line
@@ -200,6 +200,9 @@ func CheckConfig(logger *zap.Logger, config Config) map[string]string { 
			logger.Fatal("Bad database connection URL", zap.String("database.address", address), zap.Error(err))

		}

	}

	if config.GetDatabase().DnsScanIntervalSec < 1 {

		logger.Fatal("Database DNS scan interval seconds must be > 0", zap.Int("database.dns_scan_interval_sec", config.GetDatabase().DnsScanIntervalSec))

	}

	if config.GetRuntime().GetLuaMinCount() < 0 {

		logger.Fatal("Minimum Lua runtime instance count must be >= 0", zap.Int("runtime.lua_min_count", config.GetRuntime().GetLuaMinCount()))

	}
@@ -693,6 +696,7 @@ type DatabaseConfig struct { 
	ConnMaxLifetimeMs  int      `yaml:"conn_max_lifetime_ms" json:"conn_max_lifetime_ms" usage:"Time in milliseconds to reuse a database connection before the connection is killed and a new one is created. Default 3600000 (1 hour)."`

	MaxOpenConns       int      `yaml:"max_open_conns" json:"max_open_conns" usage:"Maximum number of allowed open connections to the database. Default 100."`

	MaxIdleConns       int      `yaml:"max_idle_conns" json:"max_idle_conns" usage:"Maximum number of allowed open but unused connections to the database. Default 100."`

	DnsScanIntervalSec int      `yaml:"dns_scan_interval_sec" json:"dns_scan_interval_sec" usage:"Number of seconds between scans looking for DNS resolution changes for the database hostname. Default 60."`

}



// NewDatabaseConfig creates a new DatabaseConfig struct.
@@ -702,6 +706,7 @@ func NewDatabaseConfig() *DatabaseConfig { 
		ConnMaxLifetimeMs:  3600000,

		MaxOpenConns:       100,

		MaxIdleConns:       100,

		DnsScanIntervalSec: 60,

	}

}

server/db.go

+14 −7
Original line number Diff line number Diff line
@@ -84,7 +84,7 @@ func DbConnect(ctx context.Context, logger *zap.Logger, config Config) (*sql.DB, 


	// Periodically check database hostname for underlying address changes.

	go func() {

		ticker := time.NewTicker(1 * time.Minute)

		ticker := time.NewTicker(time.Duration(config.GetDatabase().DnsScanIntervalSec) * time.Second)

		for {

			select {

			case <-ctx.Done():
@@ -118,6 +118,12 @@ func DbConnect(ctx context.Context, logger *zap.Logger, config Config) (*sql.DB, 
					break

				}



				startTime := time.Now().UTC()

				logger.Warn("Database starting rotation of all connections due to address change",

					zap.Int("count", config.GetDatabase().MaxOpenConns),

					zap.Strings("previous", resolvedAddr),

					zap.Strings("updated", newResolvedAddr))



				// Changes found. Drain the pool and allow the database driver to open fresh connections.

				// Rely on the database driver to re-do its own hostname to address resolution.

				var acquired int
@@ -137,11 +143,6 @@ func DbConnect(ctx context.Context, logger *zap.Logger, config Config) (*sql.DB, 
					conns = append(conns, conn)

				}



				logger.Warn("Database rotating all open connections due to address change",

					zap.Int("count", len(conns)),

					zap.Strings("previous", resolvedAddr),

					zap.Strings("updated", newResolvedAddr))



				resolvedAddr = newResolvedAddr

				resolvedAddrMap = newResolvedAddrMap

				for _, conn := range conns {
@@ -161,6 +162,12 @@ func DbConnect(ctx context.Context, logger *zap.Logger, config Config) (*sql.DB, 
						logger.Error("Error releasing database connection", zap.Error(err))

					}

				}



				logger.Warn("Database finished rotation of all connections due to address change",

					zap.Int("count", len(conns)),

					zap.Strings("previous", resolvedAddr),

					zap.Strings("updated", newResolvedAddr),

					zap.Duration("elapsed_duration", time.Now().UTC().Sub(startTime)))

			}

		}

	}()
@@ -173,7 +180,7 @@ func dbResolveAddress(ctx context.Context, logger *zap.Logger, host string) ([]s 
	defer resolveCtxCancelFn()

	addr, err := net.DefaultResolver.LookupHost(resolveCtx, host)

	if err != nil {

		logger.Debug("Error resolving database address", zap.String("host", host), zap.Error(err))

		logger.Debug("Error resolving database address, using previously resolved address", zap.String("host", host), zap.Error(err))

		return nil, nil

	}

	addrMap := make(map[string]struct{}, len(addr))