Commit 3afb6cd2 authored by Mo Firouz's avatar Mo Firouz Committed by Andrei Mihu
Browse files

Add Stackdriver and Prometheus monitoring support. (#178)

parent 7394445a
Loading
Loading
Loading
Loading
+147 −24
Original line number Diff line number Diff line
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.


[[projects]]
  name = "cloud.google.com/go"
  packages = [
    "compute/metadata",
    "internal/version",
    "monitoring/apiv3",
    "trace/apiv2"
  ]
  revision = "20d4028b8a750c2aca76bf9fefa8ed2d0109b573"
  version = "v0.19.0"

[[projects]]
  branch = "master"
  name = "github.com/beorn7/perks"
  packages = ["quantile"]
  revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9"

[[projects]]
  name = "github.com/davecgh/go-spew"
  packages = ["spew"]
@@ -10,8 +27,8 @@
[[projects]]
  name = "github.com/dgrijalva/jwt-go"
  packages = ["."]
  revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29"
  version = "v3.1.0"
  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
  version = "v3.2.0"

[[projects]]
  name = "github.com/go-yaml/yaml"
@@ -40,6 +57,12 @@
  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
  version = "v1.0.0"

[[projects]]
  name = "github.com/googleapis/gax-go"
  packages = ["."]
  revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f"
  version = "v2.0.0"

[[projects]]
  name = "github.com/gorhill/cronexpr"
  packages = ["."]
@@ -87,6 +110,12 @@
  ]
  revision = "83612a56d3dd153a94a629cd64925371c9adad78"

[[projects]]
  name = "github.com/matttproud/golang_protobuf_extensions"
  packages = ["pbutil"]
  revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
  version = "v1.0.0"

[[projects]]
  name = "github.com/pkg/errors"
  packages = ["."]
@@ -99,6 +128,41 @@
  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
  version = "v1.0.0"

[[projects]]
  name = "github.com/prometheus/client_golang"
  packages = [
    "prometheus",
    "prometheus/promhttp"
  ]
  revision = "d49167c4b9f3c4451707560c5c71471ff5291aaa"

[[projects]]
  branch = "master"
  name = "github.com/prometheus/client_model"
  packages = ["go"]
  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"

[[projects]]
  branch = "master"
  name = "github.com/prometheus/common"
  packages = [
    "expfmt",
    "internal/bitbucket.org/ww/goautoneg",
    "model"
  ]
  revision = "e4aa40a9169a88835b849a6efb71e05dc04b88f0"

[[projects]]
  branch = "master"
  name = "github.com/prometheus/procfs"
  packages = [
    ".",
    "internal/util",
    "nfs",
    "xfs"
  ]
  revision = "54d17b57dd7d4a3aa092476596b3f8a933bde349"

[[projects]]
  name = "github.com/rubenv/sql-migrate"
  packages = [
@@ -112,20 +176,9 @@
  packages = ["."]
  revision = "063359185d32c6b045fa171ad7033ea545864fa1"

[[projects]]
  name = "github.com/stretchr/objx"
  packages = ["."]
  revision = "facf9a85c22f48d2f52f2380e4efce1768749a89"
  version = "v0.1"

[[projects]]
  name = "github.com/stretchr/testify"
  packages = [
    ".",
    "assert",
    "http",
    "mock"
  ]
  packages = ["assert"]
  revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
  version = "v1.1.4"

@@ -142,17 +195,25 @@
[[projects]]
  name = "go.opencensus.io"
  packages = [
    "exporter/prometheus",
    "exporter/stackdriver",
    "internal",
    "internal/tagencoding",
    "plugin/grpc",
    "plugin/grpc/grpcstats",
    "plugin/grpc/grpctrace",
    "plugin/ocgrpc",
    "plugin/ochttp",
    "plugin/ochttp/propagation/b3",
    "plugin/ochttp/propagation/google",
    "stats",
    "stats/internal",
    "stats/view",
    "tag",
    "trace",
    "trace/propagation"
    "trace/propagation",
    "zpages",
    "zpages/internal"
  ]
  revision = "7e6c5d736280267bc85cfd63b23507c4889ef82e"
  version = "v0.2.0"
  revision = "983446b8dae3871316dc8610f7aa61e160b50b31"
  version = "v0.5.0"

[[projects]]
  name = "go.uber.org/atomic"
@@ -192,6 +253,7 @@
  name = "golang.org/x/net"
  packages = [
    "context",
    "context/ctxhttp",
    "http2",
    "http2/hpack",
    "idna",
@@ -201,6 +263,24 @@
  ]
  revision = "2fb46b16b8dda405028c50f7c7f0f9dd1fa6bfb1"

[[projects]]
  branch = "master"
  name = "golang.org/x/oauth2"
  packages = [
    ".",
    "google",
    "internal",
    "jws",
    "jwt"
  ]
  revision = "fdc9e635145ae97e6c2cb777c48305600cf515cb"

[[projects]]
  branch = "master"
  name = "golang.org/x/sync"
  packages = ["semaphore"]
  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"

[[projects]]
  branch = "master"
  name = "golang.org/x/text"
@@ -222,12 +302,53 @@
  ]
  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"

[[projects]]
  branch = "master"
  name = "google.golang.org/api"
  packages = [
    "googleapi/transport",
    "internal",
    "iterator",
    "option",
    "support/bundler",
    "transport",
    "transport/grpc",
    "transport/http"
  ]
  revision = "55e9fb4044f4757138d4273ace23060d022d18f9"

[[projects]]
  name = "google.golang.org/appengine"
  packages = [
    ".",
    "internal",
    "internal/app_identity",
    "internal/base",
    "internal/datastore",
    "internal/log",
    "internal/modules",
    "internal/remote_api",
    "internal/socket",
    "internal/urlfetch",
    "socket",
    "urlfetch"
  ]
  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
  version = "v1.0.0"

[[projects]]
  branch = "master"
  name = "google.golang.org/genproto"
  packages = [
    "googleapis/api/annotations",
    "googleapis/rpc/status"
    "googleapis/api/distribution",
    "googleapis/api/label",
    "googleapis/api/metric",
    "googleapis/api/monitoredres",
    "googleapis/devtools/cloudtrace/v2",
    "googleapis/monitoring/v3",
    "googleapis/rpc/status",
    "protobuf/field_mask"
  ]
  revision = "4eb30f4778eed4c258ba66527a0d4f9ec8a36c45"

@@ -241,8 +362,10 @@
    "codes",
    "connectivity",
    "credentials",
    "credentials/oauth",
    "encoding",
    "encoding/gzip",
    "encoding/proto",
    "grpclb/grpc_lb_v1/messages",
    "grpclog",
    "internal",
@@ -258,8 +381,8 @@
    "tap",
    "transport"
  ]
  revision = "6b51017f791ae1cfbec89c52efdf444b13b550ef"
  version = "v1.9.2"
  revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655"
  version = "v1.10.0"

[[projects]]
  name = "gopkg.in/gorp.v1"
@@ -270,6 +393,6 @@
[solve-meta]
  analyzer-name = "dep"
  analyzer-version = 1
  inputs-digest = "252246cd77180cbf3e076b45d44f72cc66d42f3c076c247e3fe6e5c32c5107f1"
  inputs-digest = "d34f7dd83d8d4cdaa558556d02455ac16c463f0e61afdd76ecce2729e2486235"
  solver-name = "gps-cdcl"
  solver-version = 1
+7 −7
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@

[[constraint]]
  name = "google.golang.org/grpc"
  version = "~1.9.0"
  version = "~1.10.0"

[[constraint]]
  name = "github.com/go-yaml/yaml"
@@ -28,11 +28,11 @@

[[constraint]]
  name = "github.com/gorilla/mux"
  version = "~1.6.0"
  version = "~1.6.1"

[[constraint]]
  name = "github.com/dgrijalva/jwt-go"
  version = "~3.1.0"
  version = "~3.2.0"

[[constraint]]
  name = "github.com/gorilla/websocket"
@@ -50,14 +50,14 @@
  name = "golang.org/x/crypto"
  revision = "1875d0a70c90e57f11972aefd42276df65e895b9"

[[constraint]]
  name = "go.opencensus.io"
  version = "0.2.0"

[[constraint]]
  name = "github.com/gobuffalo/packr"
  revision = "6434a292ac52e6964adebfdce3f9ce6d9f16be01"

[[override]]
  name = "github.com/prometheus/client_golang"
  revision = "d49167c4b9f3c4451707560c5c71471ff5291aaa"

[[constraint]]
  name = "github.com/stretchr/testify"
  version = "~1.1.4"
+4 −2
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ import (
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"

@@ -32,7 +33,6 @@ import (
	"github.com/heroiclabs/nakama/social"
	_ "github.com/lib/pq"
	"go.uber.org/zap"
	"sync"
)

var (
@@ -108,7 +108,8 @@ func main() {
	}
	runtimePool := server.NewRuntimePool(jsonLogger, multiLogger, db, config, socialClient, sessionRegistry, matchRegistry, tracker, router, stdLibs, modules, regRPC, once)
	pipeline := server.NewPipeline(config, db, sessionRegistry, matchRegistry, tracker, router, runtimePool)
	apiServer := server.StartApiServer(jsonLogger, db, jsonpbMarshaler, jsonpbUnmarshaler, config, socialClient, sessionRegistry, matchRegistry, tracker, router, pipeline, runtimePool)
	metrics := server.NewMetrics(multiLogger, config)
	apiServer := server.StartApiServer(jsonLogger, multiLogger, db, jsonpbMarshaler, jsonpbUnmarshaler, config, socialClient, sessionRegistry, matchRegistry, tracker, router, pipeline, runtimePool)

	// Respect OS stop signals.
	c := make(chan os.Signal, 2)
@@ -122,6 +123,7 @@ func main() {

	// Gracefully stop server components.
	apiServer.Stop()
	metrics.Stop(jsonLogger)
	matchRegistry.Stop()
	tracker.Stop()
	sessionRegistry.Stop()
+26 −12
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import (
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/dgrijalva/jwt-go"
	"github.com/golang/protobuf/jsonpb"
@@ -32,7 +33,7 @@ import (
	"github.com/heroiclabs/nakama/api"
	"github.com/heroiclabs/nakama/social"
	"github.com/satori/go.uuid"
	ocgrpc "go.opencensus.io/plugin/grpc"
	"go.opencensus.io/plugin/ocgrpc"
	"go.uber.org/zap"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
@@ -60,9 +61,10 @@ type ApiServer struct {
	grpcGatewayServer *http.Server
}

func StartApiServer(logger *zap.Logger, db *sql.DB, jsonpbMarshaler *jsonpb.Marshaler, jsonpbUnmarshaler *jsonpb.Unmarshaler, config Config, socialClient *social.Client, sessionRegistry *SessionRegistry, matchRegistry MatchRegistry, tracker Tracker, router MessageRouter, pipeline *pipeline, runtimePool *RuntimePool) *ApiServer {
func StartApiServer(logger *zap.Logger, multiLogger *zap.Logger, db *sql.DB, jsonpbMarshaler *jsonpb.Marshaler, jsonpbUnmarshaler *jsonpb.Unmarshaler, config Config, socialClient *social.Client, sessionRegistry *SessionRegistry, matchRegistry MatchRegistry, tracker Tracker, router MessageRouter, pipeline *pipeline, runtimePool *RuntimePool) *ApiServer {
	grpcServer := grpc.NewServer(
		grpc.StatsHandler(ocgrpc.NewServerStatsHandler()),
		grpc.StatsHandler(&ocgrpc.ServerHandler{IsPublicEndpoint: true}),
		grpc.MaxRecvMsgSize(int(config.GetSocket().MaxMessageSizeBytes)),
		grpc.UnaryInterceptor(SecurityInterceptorFunc(logger, config)),
	)

@@ -80,14 +82,15 @@ func StartApiServer(logger *zap.Logger, db *sql.DB, jsonpbMarshaler *jsonpb.Mars

	// Register and start GRPC server.
	api.RegisterNakamaServer(grpcServer, s)
	multiLogger.Info("Starting API server to server gRPC requests", zap.Int("port", config.GetSocket().Port))
	go func() {
		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", config.GetSocket().Port))
		if err != nil {
			logger.Fatal("API Server listener failed to start", zap.Error(err))
			multiLogger.Fatal("API Server listener failed to start", zap.Error(err))
		}

		if err := grpcServer.Serve(listener); err != nil {
			logger.Fatal("API Server listener failed", zap.Error(err))
			multiLogger.Fatal("API Server listener failed", zap.Error(err))
		}
	}()

@@ -96,9 +99,13 @@ func StartApiServer(logger *zap.Logger, db *sql.DB, jsonpbMarshaler *jsonpb.Mars
	ctx := context.Background()
	grpcGateway := runtime.NewServeMux()
	dialAddr := fmt.Sprintf("127.0.0.1:%d", config.GetSocket().Port)
	opts := []grpc.DialOption{grpc.WithInsecure()}
	opts := []grpc.DialOption{
		//TODO (mo, zyro): Do we need to pass the statsHandler here as well?
		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(config.GetSocket().MaxMessageSizeBytes))),
		grpc.WithInsecure(),
	}
	if err := api.RegisterNakamaHandlerFromEndpoint(ctx, grpcGateway, dialAddr, opts); err != nil {
		logger.Fatal("API Server gateway registration failed", zap.Error(err))
		multiLogger.Fatal("API Server gateway registration failed", zap.Error(err))
	}

	grpcGatewayRouter := mux.NewRouter()
@@ -106,8 +113,10 @@ func StartApiServer(logger *zap.Logger, db *sql.DB, jsonpbMarshaler *jsonpb.Mars
	grpcGatewayRouter.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }).Methods("GET")
	grpcGatewayRouter.HandleFunc("/ws", NewSocketWsAcceptor(logger, config, sessionRegistry, tracker, jsonpbMarshaler, jsonpbUnmarshaler, pipeline))
	// TODO restore when admin endpoints are available.
	// grpcGatewayRouter.HandleFunc("/metrics", zpages.RpczHandler)
	// grpcGatewayRouter.HandleFunc("/trace", zpages.TracezHandler)
	//grpcGatewayRouter.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
	//	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	//	zpages.WriteHTMLRpczPage(w)
	//})
	// Default to passing request to GRPC Gateway. Enable compression on gateway responses.
	handlerWithGzip := handlers.CompressHandler(grpcGateway)
	grpcGatewayRouter.NewRoute().Handler(handlerWithGzip)
@@ -121,11 +130,16 @@ func StartApiServer(logger *zap.Logger, db *sql.DB, jsonpbMarshaler *jsonpb.Mars
	// Set up and start GRPC Gateway server.
	s.grpcGatewayServer = &http.Server{
		Addr:         fmt.Sprintf(":%d", config.GetSocket().Port-1),
		ReadTimeout:  time.Millisecond * time.Duration(int64(config.GetSocket().ReadTimeoutMs)),
		WriteTimeout: time.Millisecond * time.Duration(int64(config.GetSocket().WriteTimeoutMs)),
		IdleTimeout:  time.Millisecond * time.Duration(int64(config.GetSocket().IdeaTimeoutMs)),
		Handler:      handlerWithCORS,
	}

	multiLogger.Info("Starting API server to server HTTP requests", zap.Int("port", config.GetSocket().Port-1))
	go func() {
		if err := s.grpcGatewayServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			logger.Fatal("API Server gateway listener failed", zap.Error(err))
			multiLogger.Fatal("API Server gateway listener failed", zap.Error(err))
		}
	}()

+36 −5
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ type Config interface {
	GetName() string
	GetDataDir() string
	GetLog() *LogConfig
	GetMetrics() *MetricsConfig
	GetSession() *SessionConfig
	GetSocket() *SocketConfig
	GetDatabase() *DatabaseConfig
@@ -117,6 +118,7 @@ type config struct {
	Config   string          `yaml:"config" json:"config" usage:"The absolute file path to configuration YAML file."`
	Datadir  string          `yaml:"data_dir" json:"data_dir" usage:"An absolute path to a writeable folder where Nakama will store its data."`
	Log      *LogConfig      `yaml:"log" json:"log" usage:"Log levels and output."`
	Metrics  *MetricsConfig  `yaml:"metrics" json:"metrics" usage:"Metrics settings."`
	Session  *SessionConfig  `yaml:"session" json:"session" usage:"Session authentication settings."`
	Socket   *SocketConfig   `yaml:"socket" json:"socket" usage:"Socket configuration."`
	Database *DatabaseConfig `yaml:"database" json:"database" usage:"Database connection settings."`
@@ -133,6 +135,7 @@ func NewConfig() *config {
		Name:     nodeName,
		Datadir:  dataDirectory,
		Log:      NewLogConfig(),
		Metrics:  NewMetricsConfig(),
		Session:  NewSessionConfig(),
		Socket:   NewSocketConfig(),
		Database: NewDatabaseConfig(),
@@ -153,6 +156,10 @@ func (c *config) GetLog() *LogConfig {
	return c.Log
}

func (c *config) GetMetrics() *MetricsConfig {
	return c.Metrics
}

func (c *config) GetSession() *SessionConfig {
	return c.Session
}
@@ -192,6 +199,24 @@ func NewLogConfig() *LogConfig {
	}
}

// MetricsConfig is configuration relevant to metrics capturing and output.
type MetricsConfig struct {
	ReportingFreqSec     int    `yaml:"reporting_freq_sec" json:"reporting_freq_sec" usage:"Frequency of metrics exports. Default is 1 second."`
	StackdriverProjectID string `yaml:"stackdriver_projectid" json:"stackdriver_projectid" usage:"This is the identifier of the Stackdriver project the server is uploading the stats data to. Setting this enabled metrics to be exported to Stackdriver."`
	Namespace            string `yaml:"namespace" json:"namespace" usage:"Namespace for Prometheus or prefix for Stackdriver metrics. It will always prepend node name."`
	PrometheusPort       int    `yaml:"prometheus_port" json:"prometheus_port" usage:"Port to expose Prometheus. If '0' Prometheus exports are disabled."`
}

// NewMetricsConfig creates a new MatricsConfig struct.
func NewMetricsConfig() *MetricsConfig {
	return &MetricsConfig{
		ReportingFreqSec:     1,
		StackdriverProjectID: "",
		Namespace:            "",
		PrometheusPort:       0,
	}
}

// SessionConfig is configuration relevant to the session.
type SessionConfig struct {
	EncryptionKey  string `yaml:"encryption_key" json:"encryption_key" usage:"The encryption key used to produce the client token."`
@@ -210,11 +235,14 @@ func NewSessionConfig() *SessionConfig {
type SocketConfig struct {
	ServerKey           string `yaml:"server_key" json:"server_key" usage:"Server key to use to establish a connection to the server."`
	Port                int    `yaml:"port" json:"port" usage:"The port for accepting connections from the client, listening on all interfaces."`
	MaxMessageSizeBytes int64  `yaml:"max_message_size_bytes" json:"max_message_size_bytes" usage:"Maximum amount of data in bytes allowed to be read from the client socket per message."`
	WriteWaitMs         int    `yaml:"write_wait_ms" json:"write_wait_ms" usage:"Time in milliseconds to wait for an ack from the client when writing data."`
	PongWaitMs          int    `yaml:"pong_wait_ms" json:"pong_wait_ms" usage:"Time in milliseconds to wait between pong messages received from the client."`
	PingPeriodMs        int    `yaml:"ping_period_ms" json:"ping_period_ms" usage:"Time in milliseconds to wait between sending ping messages to the client. This value must be less than the pong_wait_ms."`
	OutgoingQueueSize   int    `yaml:"outgoing_queue_size" json:"outgoing_queue_size" usage:"The maximum number of messages waiting to be sent to the client. If this is exceeded the client is considered too slow and will disconnect."`
	MaxMessageSizeBytes int64  `yaml:"max_message_size_bytes" json:"max_message_size_bytes" usage:"Maximum amount of data in bytes allowed to be read from the client socket per message. Used for real-time, gRPC and HTTP connections."`
	ReadTimeoutMs       int    `yaml:"read_timeout_ms" json:"read_timeout_ms" usage:"Maximum duration in milliseconds for reading the entire request. Used for HTTP connections."`
	WriteTimeoutMs      int    `yaml:"write_timeout_ms" json:"write_timeout_ms" usage:"Maximum duration in milliseconds before timing out writes of the response. Used for HTTP connections."`
	IdeaTimeoutMs       int    `yaml:"idle_timeout_ms" json:"idle_timeout_ms" usage:"Maximum amount of time in milliseconds to wait for the next request when keep-alives are enabled. Used for HTTP connections."`
	WriteWaitMs         int    `yaml:"write_wait_ms" json:"write_wait_ms" usage:"Time in milliseconds to wait for an ack from the client when writing data. Used for real-time connections."`
	PongWaitMs          int    `yaml:"pong_wait_ms" json:"pong_wait_ms" usage:"Time in milliseconds to wait between pong messages received from the client. Used for real-time connections."`
	PingPeriodMs        int    `yaml:"ping_period_ms" json:"ping_period_ms" usage:"Time in milliseconds to wait between sending ping messages to the client. This value must be less than the pong_wait_ms. Used for real-time connections."`
	OutgoingQueueSize   int    `yaml:"outgoing_queue_size" json:"outgoing_queue_size" usage:"The maximum number of messages waiting to be sent to the client. If this is exceeded the client is considered too slow and will disconnect. Used when processing real-time connections."`
	SSLCertificate      string `yaml:"ssl_certificate" json:"ssl_certificate" usage:"Path to certificate file if you want the server to use SSL directly. Must also supply ssl_private_key"`
	SSLPrivateKey       string `yaml:"ssl_private_key" json:"ssl_private_key" usage:"Path to private key file if you want the server to use SSL directly. Must also supply ssl_certificate"`
}
@@ -225,6 +253,9 @@ func NewSocketConfig() *SocketConfig {
		ServerKey:           "defaultkey",
		Port:                7350,
		MaxMessageSizeBytes: 2048,
		ReadTimeoutMs:       10 * 1000,
		WriteTimeoutMs:      10 * 1000,
		IdeaTimeoutMs:       60 * 1000,
		WriteWaitMs:         5000,
		PongWaitMs:          10000,
		PingPeriodMs:        8000,
Loading