diff --git a/conn.go b/conn.go
index da84976..f8f6057 100644
--- a/conn.go
+++ b/conn.go
@@ -41,10 +41,15 @@ func (c *serverConn) Close() (err error) {
 }
 
 func (c *serverConn) updateDeadline() {
-	idleDeadline := time.Now().Add(c.idleTimeout)
-	if idleDeadline.Unix() < c.maxDeadline.Unix() {
-		c.Conn.SetDeadline(idleDeadline)
-	} else {
+	switch {
+	case c.idleTimeout > 0:
+		idleDeadline := time.Now().Add(c.idleTimeout)
+		if idleDeadline.Unix() < c.maxDeadline.Unix() {
+			c.Conn.SetDeadline(idleDeadline)
+			return
+		}
+		fallthrough
+	default:
 		c.Conn.SetDeadline(c.maxDeadline)
 	}
 }
diff --git a/server.go b/server.go
index 63a380c..6f13ad5 100644
--- a/server.go
+++ b/server.go
@@ -209,7 +209,7 @@ func (srv *Server) handleConn(newConn net.Conn) {
 		idleTimeout:   srv.IdleTimeout,
 		closeCanceler: cancel,
 	}
-	if int64(srv.MaxTimeout) > 0 {
+	if srv.MaxTimeout > 0 {
 		conn.maxDeadline = time.Now().Add(srv.MaxTimeout)
 	}
 	defer conn.Close()