diff --git a/server.go b/server.go
index c6efc69..bf39f84 100644
--- a/server.go
+++ b/server.go
@@ -33,18 +33,11 @@ type Server struct {
 	IdleTimeout time.Duration // connection timeout when no activity, none if empty
 	MaxTimeout  time.Duration // absolute connection timeout, none if empty
 
-	// The allowed key exchanges algorithms. If unspecified then a
-	// default set of algorithms is used. Most users should not need to set
-	// this.
-	KeyExchanges []string
-
-	// The allowed cipher algorithms. If unspecified then a sensible
-	// default is used. Most users should not need to set this.
-	Ciphers []string
-
-	// The allowed MAC algorithms. If unspecified then a sensible default
-	// is used. Most users should not need to set this.
-	MACs []string
+	// Internal x/crypto/ssh config. Note that a number of values in this struct
+	// are overwritten every time a connection starts, so only use this if you
+	// know what you're doing and absolutely need to change the internal config
+	// values.
+	BaseConfig *gossh.ServerConfig
 
 	channelHandlers map[string]channelHandler
 
@@ -71,19 +64,13 @@ func (srv *Server) ensureHostSigner() error {
 }
 
 func (srv *Server) config(ctx Context) *gossh.ServerConfig {
-	config := &gossh.ServerConfig{}
-	if len(srv.KeyExchanges) > 0 {
-		config.KeyExchanges = make([]string, len(srv.KeyExchanges))
-		copy(srv.KeyExchanges, config.KeyExchanges)
-	}
-	if len(srv.Ciphers) > 0 {
-		config.Ciphers = make([]string, len(srv.Ciphers))
-		copy(srv.Ciphers, config.Ciphers)
-	}
-	if len(srv.MACs) > 0 {
-		config.MACs = make([]string, len(srv.MACs))
-		copy(srv.MACs, config.MACs)
+	// Use the provided base config if set, otherwise default to an empty
+	// config.
+	config := srv.BaseConfig
+	if config == nil {
+		config = &gossh.ServerConfig{}
 	}
+
 	for _, signer := range srv.HostSigners {
 		config.AddHostKey(signer)
 	}
@@ -112,6 +99,7 @@ func (srv *Server) config(ctx Context) *gossh.ServerConfig {
 			return ctx.Permissions().Permissions, nil
 		}
 	}
+
 	return config
 }