Grpc-gateway: how to do tls auth in grpc+grpc-gateway

[vtolstov]

I'm already have grpc server with my own letsencrypt cert behind Nginx:
nginx listens ssl, and proxy_pass to https grpc-gateway

grpc server listens on int_ip:7777, nginx listens on ext_ip: 443, hostname - external domain name on nginx, endpoint intenal listen addr - int_ip:7777

grpc code stuff:

certPool, err := certPool() // add fullchain.pem from letsencrypt
if err != nil {
  return nil, err
}
 opts = append(opts, grpc.Creds(credentials.NewClientTLSFromCert(certPool, hostname)))
}
opts = append(opts, grpc.StreamInterceptor(grpc_auth.StreamServerInterceptor(authHandlerFunc)))
opts = append(opts, grpc.UnaryInterceptor(grpc_auth.UnaryServerInterceptor(authHandlerFunc)))
grpcs := grpc.NewServer(opts...)

gw stuff:

   opts = append(opts, grpc.WithTimeout(2*time.Second), grpc.WithBlock())

        certPool, err := certPool()
        if err != nil {
                return nil, err
        }

                certificate, err := tls.LoadX509KeyPair(crt, key)
                if err != nil {
                        return nil, fmt.Errorf("could not load server key pair: %s", err)
                }
                creds := credentials.NewTLS(&tls.Config{
                        ServerName:   hostname,
                        Certificates: []tls.Certificate{certificate},
                        RootCAs:      certPool,
                })
                opts = append(opts, grpc.WithTransportCredentials(creds))
 conn, err := grpc.Dial(endpoint, opts...)

main stuff:

        certificate, err := tls.LoadX509KeyPair(crt, key)
        if err != nil {
                return nil, err
        }

        tlsConf := &tls.Config{
                Certificates: []tls.Certificate{certificate},
                Rand:         rand.Reader,
        }

ln, err := net.Listen("tcp", "127.0.0.1:7777")
tls.NewListener(ln, tcpConf)

       cm := cmux.New(ln)
        grpcLn := cm.MatchWithWriters(cmux.HTTP2MatchHeaderFieldPrefixSendSettings("content-type", "application/grpc"))
        //wsLn := cm.Match(cmux.HTTP1HeaderField("Upgrade", "websocket"))
        restLn := cm.Match(cmux.HTTP1Fast())

        grpcs, err := prepareGRPC(ctx)
        if err != nil {
                log.Fatal(err)
        }

        rests, err := prepareREST(ctx)
        if err != nil {
                log.Fatal(err)
        }

        go func() {
                if err = grpcs.Serve(grpcLn); err != cmux.ErrListenerClosed {
                        log.Fatal(err)
                }
        }()
        go func() {
                if err = rests.Serve(restLn); err != cmux.ErrListenerClosed {
                        log.Fatal(err)
                }
        }()

        if err := cm.Serve(); !strings.Contains(err.Error(), "use of closed network connection") {
                log.Fatal(err)
        }

when i'm try to start my app i have context deadline exceeded because grpc-gateway can't connect to grpc server

[vtolstov]

Without tls all works fine with cmux, i can connect without certs to grpc server and do some rest requests, but with tls enabled i'm always have timeouts

[vtolstov]

i think that me question is:
does it possible to use the same cert on client (gateway) and on server ?

[johanbrandhorst]

mTLS between the grpc-gateway client and the gRPC server should absolutely be possible. You have to turn off client authentication for your gateway though, unless you expect those clients to also present a certificate. https://github.com/gogo/grpc-example/blob/master/main.go shows an example of how to do server auth, and adding client auth on top shouldn't be much work.

[vtolstov]

I found the root of my issue - cmux.
I need to run both grpc and rest on the same port. In case of plain tcp - all fine, cmux works by query request headers. In case of crypt - i need to pass to grpc server plain tcp listener, but for rest i need to pass tls encrypted listener.

[vtolstov]

So my new question - does have somebody already knows how to do encrypted grpc+rest with cmux.

[vtolstov]

i'm solve. With plain mode i'm use cmux to get rest and grpc on the same port.
For tls mode i'm use another example from @philips

[achew22]

Could you post the code you ended up using in here? It might be useful to others in the future

[vtolstov]

        tcpl, err := net.Listen("tcp", *endpoint)
        if err != nil {
                log.Fatalf("%s", err)
        }
        defer tcpl.Close()

        if !secure {
                tcpm = cmux.New(tcpl)
                //              grpcl = tcpm.MatchWithWriters(cmux.HTTP2MatchHeaderFieldPrefixSendSettings("content-type", "application/
grpc"))
                grpcl = tcpm.Match(cmux.HTTP2())
                restl = tcpm.Match(cmux.HTTP1())
        } else {
                tlsl, err := tlsListener(tcpl)
                if err != nil {
                        log.Fatal("%s", err)
                }
                grpcl = tlsl
                restl = tlsl
        }

        grpcs, err := prepareGRPC(ctx)
        if err != nil {
                log.Fatal(err)
        }
        rests, err := prepareREST(ctx, grpcs)
        if err != nil {
                log.Fatal(err)
        }

        if !secure {
                go func() {
                        if err = grpcs.Serve(grpcl); err != cmux.ErrListenerClosed {
                                log.Fatal(err)
                        }
                }()
        }
        go func() {
                if err = rests.Serve(restl); err != cmux.ErrListenerClosed {
                        log.Fatal(err)
                }
        }()

        if !secure {
                if err := tcpm.Serve(); !strings.Contains(err.Error(), "use of closed network connection") {
                        log.Fatal(err)
                }
        } else {
                select {}
        }

[vtolstov]

main part is above. may be i create some repo to put all stuff to it and provide link.

[vtolstov]

no, i don't create repo

[dayadev]

@vtolstov Thanks!

[johanbrandhorst]

I've written about gRPC client authentication which might be useful: https://jbrandhorst.com/post/grpc-auth/

[azhuo-va]

@vtolstov Can you please also post the function prepareGRPC and prepareREST? I am wondering why prepareREST needs to accept grpcs as a parameter. Thanks!

[judavi]

@vtolstov thanks!! your example helped me to solve an issue!!

[Z-a-r-a-k-i]

Hello, I tried to do same as @vtolstov, creating a tls listener but then I always get cmux match error. The only solution I found to make my backend work with grpc and oauth token was to disable the http server which is kinda sad.

Here is the code if some are interested or eventually if someone can help me solve this issue:

server.go

package dtvapi

import (
    "context"
    "crypto/rand"
    "crypto/tls"
    "errors"
    "fmt"
    "log"
    "net"
    "net/http"
    "net/http/pprof"
    "runtime/debug"
    "strings"
    "time"

    "github.com/go-chi/chi"
    "github.com/go-chi/chi/middleware"
    "github.com/gogo/gateway"
    grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
    grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
    grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
    grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"
    "github.com/grpc-ecosystem/grpc-gateway/runtime"
    "github.com/oklog/run"
    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/ext"
    "github.com/rs/cors"
    "github.com/soheilhy/cmux"
    chilogger "github.com/treastech/logger"
    "go.uber.org/zap"
    grpc "google.golang.org/grpc"
    codes "google.golang.org/grpc/codes"
    "google.golang.org/grpc/credentials"
    status "google.golang.org/grpc/status"
)

func NewServer(ctx context.Context, svc Service, opts ServerOpts) (*Server, error) {
    // assign default opts
    if opts.Logger == nil {
        opts.Logger = zap.NewNop()
    }
    if opts.CORSAllowedOrigins == "" {
        opts.CORSAllowedOrigins = "*"
    }
    if opts.Bind == "" {
        opts.Bind = ":0"
    }
    if opts.RequestTimeout == 0 {
        opts.RequestTimeout = 5 * time.Second
    }
    if opts.ShutdownTimeout == 0 {
        opts.ShutdownTimeout = 6 * time.Second
    }
    s := Server{logger: opts.Logger}

    // listener
    var err error
    s.masterListener, err = net.Listen("tcp", opts.Bind)
    if err != nil {
        return nil, err
    }
    s.cmux = cmux.New(s.masterListener)

    //tlsl := tlsListener(s.masterListener, opts.TLSCertPath, opts.TLSKeyPath)*/

    //s.grpcListener = s.cmux.Match(cmux.Any())
    //s.grpcListener = s.cmux.MatchWithWriters(cmux.HTTP2MatchHeaderFieldPrefixSendSettings("content-type", "application/grpc"))
    s.grpcListener = s.cmux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
    //s.httpListener = s.cmux.Match(cmux.HTTP1(), cmux.HTTP2())

    //s.grpcListener = tlsl
    //s.httpListener = tlsl

    // FIXME: add gRPC web support
    // FIXME: websocket

    // grpc server
    s.grpcServer, err = grpcServer(svc, opts)
    if err != nil {
        return nil, err
    }
    s.workers.Add(func() error {
        err := s.grpcServer.Serve(s.grpcListener)
        if err != cmux.ErrListenerClosed {
            return err
        }
        return nil
    }, func(error) {
        if err := s.grpcListener.Close(); err != nil {
            opts.Logger.Warn("close listener", zap.Error(err))
        }
    })

    // http server
    /*httpServer, err := httpServer(ctx, s.ListenerAddr(), opts)
    if err != nil {
        return nil, err
    }
    s.workers.Add(func() error {
        //err := httpServer.ServeTLS(s.httpListener, opts.TLSCertPath, opts.TLSKeyPath)
        err := httpServer.Serve(s.httpListener)
        if err != cmux.ErrListenerClosed {
            return err
        }
        return nil
    }, func(error) {
        ctx, cancel := context.WithTimeout(ctx, opts.ShutdownTimeout)
        if err := httpServer.Shutdown(ctx); err != nil {
            opts.Logger.Warn("shutdown HTTP server", zap.Error(err))
        }
        defer cancel()
        if err := s.httpListener.Close(); err != nil {
            opts.Logger.Warn("close listener", zap.Error(err))
        }
    })*/

    s.cmux.HandleError(func(err error) bool {
        s.logger.Warn("cmux error", zap.Error(err))
        return true
    })

    // mux
    s.workers.Add(
        func() error {
            err := s.cmux.Serve()
            return err
        },
        func(err error) {
            fmt.Println(err)
        },
    )
    return &s, nil
}

// Server is an HTTP+gRPC frontend for Service
type Server struct {
    grpcServer     *grpc.Server
    masterListener net.Listener
    grpcListener   net.Listener
    httpListener   net.Listener
    cmux           cmux.CMux
    logger         *zap.Logger
    workers        run.Group
}

type ServerOpts struct {
    Logger             *zap.Logger
    Bind               string
    CORSAllowedOrigins string
    RequestTimeout     time.Duration
    ShutdownTimeout    time.Duration
    WithPprof          bool
    Tracer             opentracing.Tracer
    TLSCertPath        string
    TLSKeyPath         string
}

func (s *Server) Run() error {
    return s.workers.Run()
}

func (s *Server) Close() {
    //go s.grpcServer.GracefulStop()
    //time.Sleep(time.Second)
    //s.grpcServer.Stop()
    s.masterListener.Close()
}

func (s *Server) ListenerAddr() string {
    return s.masterListener.Addr().String()
}

func grpcServer(svc Service, opts ServerOpts) (*grpc.Server, error) {
    authFunc := func(context.Context) (context.Context, error) {
        return nil, errors.New("grpc auth func error")
    }
    recoveryOpts := []grpc_recovery.Option{}
    if opts.Logger.Check(zap.DebugLevel, "") != nil {
        recoveryOpts = append(recoveryOpts, grpc_recovery.WithRecoveryHandlerContext(func(ctx context.Context, p interface{}) error {
            log.Println("stacktrace from panic: \n" + string(debug.Stack()))
            return status.Errorf(codes.Internal, "recover: %s", p)
        }))
    }
    serverStreamOpts := []grpc.StreamServerInterceptor{grpc_recovery.StreamServerInterceptor(recoveryOpts...)}
    serverUnaryOpts := []grpc.UnaryServerInterceptor{grpc_recovery.UnaryServerInterceptor(recoveryOpts...)}
    if opts.Tracer != nil {
        tracingOpts := []grpc_opentracing.Option{grpc_opentracing.WithTracer(opts.Tracer)}
        serverStreamOpts = append(serverStreamOpts, grpc_opentracing.StreamServerInterceptor(tracingOpts...))
        serverUnaryOpts = append(serverUnaryOpts, grpc_opentracing.UnaryServerInterceptor(tracingOpts...))
    }
    serverStreamOpts = append(
        serverStreamOpts,
        grpc_auth.StreamServerInterceptor(authFunc),
        //grpc_ctxtags.StreamServerInterceptor(),
        //grpc_zap.StreamServerInterceptor(logger),
    )
    serverUnaryOpts = append(
        serverUnaryOpts,
        grpc_auth.UnaryServerInterceptor(authFunc),
        //grpc_ctxtags.UnaryServerInterceptor(),
        //grpc_zap.UnaryServerInterceptor(logger),
    )
    if opts.Logger.Check(zap.DebugLevel, "") != nil {
        serverStreamOpts = append(serverStreamOpts, grpcServerStreamInterceptor())
        serverUnaryOpts = append(serverUnaryOpts, grpcServerUnaryInterceptor())
    }
    serverStreamOpts = append(serverStreamOpts, grpc_recovery.StreamServerInterceptor(recoveryOpts...))
    serverUnaryOpts = append(serverUnaryOpts, grpc_recovery.UnaryServerInterceptor(recoveryOpts...))
    creds, err := credentials.NewServerTLSFromFile(opts.TLSCertPath, opts.TLSKeyPath)
    if err != nil {
        log.Fatalf("Failed to generate credentials %v", err)
    }
    grpcServer := grpc.NewServer(
        grpc.Creds(creds),
        grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(serverStreamOpts...)),
        grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(serverUnaryOpts...)),
    )
    RegisterServiceServer(grpcServer, svc)

    return grpcServer, nil
}

func grpcServerStreamInterceptor() grpc.StreamServerInterceptor {
    return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
        err := handler(srv, stream)
        if err != nil {
            log.Printf("%+v", err)
        }
        return err
    }
}

func grpcServerUnaryInterceptor() grpc.UnaryServerInterceptor {
    return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
        ret, err := handler(ctx, req)
        if err != nil {
            log.Printf("%+v", err)
        }
        return ret, err
    }
}

func httpServer(ctx context.Context, serverListenerAddr string, opts ServerOpts) (*http.Server, error) {
    var err error
    logger := opts.Logger.Named("http")
    r := chi.NewRouter()
    cors := cors.New(cors.Options{
        AllowedOrigins:   strings.Split(opts.CORSAllowedOrigins, ","),
        AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
        AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
        ExposedHeaders:   []string{"Link"},
        AllowCredentials: true,
        MaxAge:           300,
    })
    r.Use(cors.Handler)
    r.Use(chilogger.Logger(logger))
    r.Use(middleware.Recoverer)
    r.Use(middleware.Timeout(opts.RequestTimeout))
    r.Use(middleware.RealIP)
    r.Use(middleware.RequestID)

    runtimeMux := runtime.NewServeMux(
        runtime.WithMarshalerOption(runtime.MIMEWildcard, &gateway.JSONPb{
            EmitDefaults: false,
            Indent:       "  ",
            OrigName:     true,
        }),
        runtime.WithProtoErrorHandler(runtime.DefaultHTTPProtoErrorHandler),
        runtime.WithIncomingHeaderMatcher(incomingHeaderMatcherFunc),
    )
    var gwmux http.Handler = runtimeMux
    dialOpts := []grpc.DialOption{grpc.WithInsecure()}
    /*creds, err := credentials.NewClientTLSFromFile(opts.TLSCertPath, "*")
    if err != nil {
        fmt.Println(err)
    }*/

    //dialOpts := []grpc.DialOption{grpc.WithTransportCredentials(creds)}
    if opts.Tracer != nil {
        var grpcGatewayTag = opentracing.Tag{Key: string(ext.Component), Value: "grpc-gateway"}
        tracingWrapper := func(h http.Handler) http.Handler {
            return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
                parentSpanContext, err := opts.Tracer.Extract(
                    opentracing.HTTPHeaders,
                    opentracing.HTTPHeadersCarrier(r.Header),
                )
                if err == nil || err == opentracing.ErrSpanContextNotFound {
                    serverSpan := opts.Tracer.StartSpan(
                        "ServeHTTP",
                        ext.RPCServerOption(parentSpanContext),
                        grpcGatewayTag,
                    )
                    r = r.WithContext(opentracing.ContextWithSpan(r.Context(), serverSpan))
                    defer serverSpan.Finish()
                }
                fmt.Println(r.Context())
                h.ServeHTTP(w, r)
            })
        }
        gwmux = tracingWrapper(gwmux)

        dialOpts = append(dialOpts,
            grpc.WithStreamInterceptor(
                grpc_opentracing.StreamClientInterceptor(
                    grpc_opentracing.WithTracer(opts.Tracer))),
            grpc.WithUnaryInterceptor(
                grpc_opentracing.UnaryClientInterceptor(
                    grpc_opentracing.WithTracer(opts.Tracer))),
        )
    }

    err = RegisterServiceHandlerFromEndpoint(ctx, runtimeMux, serverListenerAddr, dialOpts)
    if err != nil {
        return nil, err
    }

    r.Mount("/", gwmux)
    if opts.WithPprof {
        r.HandleFunc("/debug/pprof/*", pprof.Index)
        r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
        r.HandleFunc("/debug/pprof/profile", pprof.Profile)
        r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
        r.HandleFunc("/debug/pprof/trace", pprof.Trace)
    }
    http.DefaultServeMux = http.NewServeMux() // disables default handlers registered by importing net/http/pprof for security reasons
    return &http.Server{
        Addr:    opts.Bind,
        Handler: r,
    }, nil
}

func tlsListener(l net.Listener, cert string, key string) net.Listener {
    // Load certificates.
    certificate, err := tls.LoadX509KeyPair(cert, key)
    if err != nil {
        log.Panic(err)
    }

    config := &tls.Config{
        Certificates: []tls.Certificate{certificate},
        Rand:         rand.Reader,
    }

    // Create TLS listener.
    tlsl := tls.NewListener(l, config)
    return tlsl
}

[johanbrandhorst]

I think it should be possible to do gateway and grpc on the same port with cmux, note that you need to take https://github.com/soheilhy/cmux/issues/64 into account. My comment on there might be useful?