ircd.chat/ircd
11
1
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Merge pull request #1536 from slingamn/ircreader.1

Browse Source 
use the new goshuirc ircreader
This commit is contained in:
Shivaram Lingamneni 2021-02-13 21:06:42 -05:00 committed by GitHub
commit 9fd4210416
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 141 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -10,7 +10,7 @@ require (
github.com/go-sql-driver/mysql v1.5.0
github.com/go-test/deep v1.0.6 // indirect
github.com/gorilla/websocket v1.4.2
github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5
github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a
github.com/onsi/ginkgo v1.12.0 // indirect
github.com/onsi/gomega v1.9.0 // indirect
github.com/oragono/confusables v0.0.0-20201108231250-4ab98ab61fb1

4
go.sum
View File

@ -28,6 +28,10 @@ github.com/goshuirc/irc-go v0.0.0-20201228002532-4e36cb3f41f1 h1:Kyyey3K8nhx60lt
github.com/goshuirc/irc-go v0.0.0-20201228002532-4e36cb3f41f1/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5 h1:TXGvyYHJEBluqwI8d0V5/QmSnNxEYIMbfPE36B8CNK8=
github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
github.com/goshuirc/irc-go v0.0.0-20210214005848-fcaabd19f360 h1:ChbmWPZwyfgsZd6zxw7B/4hWJE7ezmb69PezUM9+YA4=
github.com/goshuirc/irc-go v0.0.0-20210214005848-fcaabd19f360/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a h1:PR1tw21nn93AwKmjEPA7IVHiT+ld9qgO1H32APCMvL0=
github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

4
irc/client.go
View File

@ -18,7 +18,9 @@ import (
"github.com/goshuirc/irc-go/ircfmt"
"github.com/goshuirc/irc-go/ircmsg"
"github.com/goshuirc/irc-go/ircreader"
ident "github.com/oragono/go-ident"
"github.com/oragono/oragono/irc/caps"
"github.com/oragono/oragono/irc/connection_limits"
"github.com/oragono/oragono/irc/flatip"
@ -689,7 +691,7 @@ func (client *Client) run(session *Session) {
} else if err != nil {
var quitMessage string
switch err {
case errReadQ, errWSBinaryMessage:
case ircreader.ErrReadQ, errWSBinaryMessage:
quitMessage = err.Error()
default:
quitMessage = "connection closed"

75
irc/ircconn.go
View File

@ -6,12 +6,12 @@ package irc
import (
"bytes"
"errors"
"io"
"net"
"unicode/utf8"
"github.com/gorilla/websocket"
"github.com/goshuirc/irc-go/ircmsg"
"github.com/goshuirc/irc-go/ircreader"
"github.com/oragono/oragono/irc/utils"
)
@ -23,7 +23,6 @@ const (
var (
crlf = []byte{'\r', '\n'}
errReadQ = errors.New("ReadQ Exceeded")
errWSBinaryMessage = errors.New("WebSocket binary messages are unsupported")
)
@ -48,17 +47,14 @@ type IRCConn interface {
type IRCStreamConn struct {
conn *utils.WrappedConn
buf []byte
start int // start of valid (i.e., read but not yet consumed) data in the buffer
end int // end of valid data in the buffer
searchFrom int // start of valid data in the buffer not yet searched for \n
eof bool
reader ircreader.IRCReader
}
func NewIRCStreamConn(conn *utils.WrappedConn) *IRCStreamConn {
return &IRCStreamConn{
conn: conn,
}
var c IRCStreamConn
c.conn = conn
c.reader.Initialize(conn.Conn, initialBufferSize, maxReadQBytes)
return &c
}
func (cc *IRCStreamConn) UnderlyingConn() *utils.WrappedConn {
@ -78,56 +74,13 @@ func (cc *IRCStreamConn) WriteLines(buffers [][]byte) (err error) {
}
func (cc *IRCStreamConn) ReadLine() ([]byte, error) {
for {
// try to find a terminated line in the buffered data already read
nlidx := bytes.IndexByte(cc.buf[cc.searchFrom:cc.end], '\n')
if nlidx != -1 {
// got a complete line
line := cc.buf[cc.start : cc.searchFrom+nlidx]
cc.start = cc.searchFrom + nlidx + 1
cc.searchFrom = cc.start
if globalUtf8EnforcementSetting && !utf8.Valid(line) {
return line, errInvalidUtf8
} else {
return line, nil
}
}
if cc.start == 0 && len(cc.buf) == maxReadQBytes {
return nil, errReadQ // out of space, can't expand or slide
}
if cc.eof {
return nil, io.EOF
}
if len(cc.buf) < maxReadQBytes && (len(cc.buf)-(cc.end-cc.start) < initialBufferSize/2) {
// allocate a new buffer, copy any remaining data
newLen := utils.RoundUpToPowerOfTwo(len(cc.buf) + 1)
if newLen > maxReadQBytes {
newLen = maxReadQBytes
} else if newLen < initialBufferSize {
newLen = initialBufferSize
}
newBuf := make([]byte, newLen)
copy(newBuf, cc.buf[cc.start:cc.end])
cc.buf = newBuf
} else if cc.start != 0 {
// slide remaining data back to the front of the buffer
copy(cc.buf, cc.buf[cc.start:cc.end])
}
cc.end = cc.end - cc.start
cc.start = 0
cc.searchFrom = cc.end
n, err := cc.conn.Read(cc.buf[cc.end:])
cc.end += n
if n != 0 && err == io.EOF {
// we may have received new \n-terminated lines, try to parse them
cc.eof = true
} else if err != nil {
return nil, err
}
line, err := cc.reader.ReadLine()
if err != nil {
return nil, err
} else if globalUtf8EnforcementSetting && !utf8.Valid(line) {
return line, errInvalidUtf8
} else {
return line, nil
}
}
@ -175,7 +128,7 @@ func (wc IRCWSConn) ReadLine() (line []byte, err error) {
return nil, errWSBinaryMessage
}
} else if err == websocket.ErrReadLimit {
return line, errReadQ
return line, ircreader.ErrReadQ
} else {
return line, err
}

135
irc/ircconn_test.go
View File

@ -1,135 +0,0 @@
// Copyright (c) 2020 Shivaram Lingamneni <slingamn@cs.stanford.edu>
// released under the MIT license
package irc
import (
"io"
"math/rand"
"net"
"reflect"
"testing"
"time"
"github.com/oragono/oragono/irc/utils"
)
// mockConn is a fake net.Conn / io.Reader that yields len(counts) lines,
// each consisting of counts[i] 'a' characters and a terminating '\n'
type mockConn struct {
counts []int
}
func min(i, j int) (m int) {
if i < j {
return i
} else {
return j
}
}
func (c *mockConn) Read(b []byte) (n int, err error) {
for len(b) > 0 {
if len(c.counts) == 0 {
return n, io.EOF
}
if c.counts[0] == 0 {
b[0] = '\n'
c.counts = c.counts[1:]
b = b[1:]
n += 1
continue
}
size := min(c.counts[0], len(b))
for i := 0; i < size; i++ {
b[i] = 'a'
}
c.counts[0] -= size
b = b[size:]
n += size
}
return n, nil
}
func (c *mockConn) Write(b []byte) (n int, err error) {
return
}
func (c *mockConn) Close() error {
c.counts = nil
return nil
}
func (c *mockConn) LocalAddr() net.Addr {
return nil
}
func (c *mockConn) RemoteAddr() net.Addr {
return nil
}
func (c *mockConn) SetDeadline(t time.Time) error {
return nil
}
func (c *mockConn) SetReadDeadline(t time.Time) error {
return nil
}
func (c *mockConn) SetWriteDeadline(t time.Time) error {
return nil
}
func newMockConn(counts []int) *utils.WrappedConn {
cpCounts := make([]int, len(counts))
copy(cpCounts, counts)
c := &mockConn{
counts: cpCounts,
}
return &utils.WrappedConn{
Conn: c,
}
}
// construct a mock reader with some number of \n-terminated lines,
// verify that IRCStreamConn can read and split them as expected
func doLineReaderTest(counts []int, t *testing.T) {
c := newMockConn(counts)
r := NewIRCStreamConn(c)
var readCounts []int
for {
line, err := r.ReadLine()
if err == nil {
readCounts = append(readCounts, len(line))
} else if err == io.EOF {
break
} else {
panic(err)
}
}
if !reflect.DeepEqual(counts, readCounts) {
t.Errorf("expected %#v, got %#v", counts, readCounts)
}
}
const (
maxMockReaderLen = 100
maxMockReaderLineLen = 4096 + 511
)
func TestLineReader(t *testing.T) {
counts := []int{44, 428, 3, 0, 200, 2000, 0, 4044, 33, 3, 2, 1, 0, 1, 2, 3, 48, 555}
doLineReaderTest(counts, t)
// fuzz
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < 1000; i++ {
countsLen := r.Intn(maxMockReaderLen) + 1
counts := make([]int, countsLen)
for i := 0; i < countsLen; i++ {
counts[i] = r.Intn(maxMockReaderLineLen)
}
doLineReaderTest(counts, t)
}
}

117
vendor/github.com/goshuirc/irc-go/ircreader/ircreader.go generated vendored Normal file
View File

@ -0,0 +1,117 @@
// Copyright (c) 2020-2021 Shivaram Lingamneni
// released under the MIT license
package ircreader
import (
"bytes"
"errors"
"io"
)
/*
IRCReader is an optimized line reader for IRC lines containing tags;
most IRC lines will not approach the maximum line length (8191 bytes
of tag data, plus 512 bytes of message data), so we want a buffered
reader that can start with a smaller buffer and expand if necessary,
while also maintaining a hard upper limit on the size of the buffer.
*/
var (
ErrReadQ = errors.New("readQ exceeded (read too many bytes without terminating newline)")
)
type IRCReader struct {
conn io.Reader
initialSize int
maxSize int
buf []byte
start int // start of valid (i.e., read but not yet consumed) data in the buffer
end int // end of valid data in the buffer
searchFrom int // start of valid data in the buffer not yet searched for \n
eof bool
}
// Returns a new *IRCReader with sane buffer size limits.
func NewIRCReader(conn io.Reader) *IRCReader {
var reader IRCReader
reader.Initialize(conn, 512, 8192+1024)
return &reader
}
// "Placement new" for an IRCReader; initializes it with custom buffer size
// limits.
func (cc *IRCReader) Initialize(conn io.Reader, initialSize, maxSize int) {
*cc = IRCReader{}
cc.conn = conn
cc.initialSize = initialSize
cc.maxSize = maxSize
}
// Blocks until a full IRC line is read, then returns it. Accepts either \n
// or \r\n as the line terminator (but not \r in isolation). Passes through
// errors from the underlying connection. Returns ErrReadQ if the buffer limit
// was exceeded without a terminating \n.
func (cc *IRCReader) ReadLine() ([]byte, error) {
for {
// try to find a terminated line in the buffered data already read
nlidx := bytes.IndexByte(cc.buf[cc.searchFrom:cc.end], '\n')
if nlidx != -1 {
// got a complete line
line := cc.buf[cc.start : cc.searchFrom+nlidx]
cc.start = cc.searchFrom + nlidx + 1
cc.searchFrom = cc.start
return line, nil
}
if cc.start == 0 && len(cc.buf) == cc.maxSize {
return nil, ErrReadQ // out of space, can't expand or slide
}
if cc.eof {
return nil, io.EOF
}
if len(cc.buf) < cc.maxSize && (len(cc.buf)-(cc.end-cc.start) < cc.initialSize/2) {
// allocate a new buffer, copy any remaining data
newLen := roundUpToPowerOfTwo(len(cc.buf) + 1)
if newLen > cc.maxSize {
newLen = cc.maxSize
} else if newLen < cc.initialSize {
newLen = cc.initialSize
}
newBuf := make([]byte, newLen)
copy(newBuf, cc.buf[cc.start:cc.end])
cc.buf = newBuf
} else if cc.start != 0 {
// slide remaining data back to the front of the buffer
copy(cc.buf, cc.buf[cc.start:cc.end])
}
cc.end = cc.end - cc.start
cc.start = 0
cc.searchFrom = cc.end
n, err := cc.conn.Read(cc.buf[cc.end:])
cc.end += n
if n != 0 && err == io.EOF {
// we may have received new \n-terminated lines, try to parse them
cc.eof = true
} else if err != nil {
return nil, err
}
}
}
// return n such that v <= n and n == 2**i for some i
func roundUpToPowerOfTwo(v int) int {
// http://graphics.stanford.edu/~seander/bithacks.html
v -= 1
v |= v >> 1
v |= v >> 2
v |= v >> 4
v |= v >> 8
v |= v >> 16
return v + 1
}

3
vendor/modules.txt vendored
View File

@ -21,10 +21,11 @@ github.com/go-sql-driver/mysql
# github.com/gorilla/websocket v1.4.2
## explicit
github.com/gorilla/websocket
# github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5
# github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a
## explicit
github.com/goshuirc/irc-go/ircfmt
github.com/goshuirc/irc-go/ircmsg
github.com/goshuirc/irc-go/ircreader
# github.com/onsi/ginkgo v1.12.0
## explicit
# github.com/onsi/gomega v1.9.0