open-nomad/helper/escapingio/reader_test.go

package escapingio

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"testing"
	"testing/iotest"
	"testing/quick"
	"time"
	"unicode"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestEscapingReader_Static(t *testing.T) {
	cases := []struct {
		input    string
		expected string
		escaped  string
	}{
		{"hello", "hello", ""},
		{"he\nllo", "he\nllo", ""},
		{"he~.lo", "he~.lo", ""},
		{"he\n~.rest", "he\nrest", "."},
		{"he\n~.r\n~.est", "he\nr\nest", ".."},
		{"he\n~~r\n~~est", "he\n~r\n~est", ""},
		{"he\n~~r\n~.est", "he\n~r\nest", "."},
		{"he\nr~~est", "he\nr~~est", ""},
		{"he\nr\n~qest", "he\nr\n~qest", "q"},
		{"he\nr\r~qe\r~.st", "he\nr\r~qe\rst", "q."},
		{"~q", "~q", "q"},
		{"~.", "", "."},
		{"m~.", "m~.", ""},
		{"\n~.", "\n", "."},
		{"~", "~", ""},
		{"\r~.", "\r", "."},
		{"b\n~\n~.q", "b\n~\nq", "."},
	}

	for _, c := range cases {
		t.Run("validate naive implementation", func(t *testing.T) {
			h := &testHandler{}

			processed := naiveEscapeCharacters(c.input, '~', h.handler)
			require.Equal(t, c.expected, processed)
			require.Equal(t, c.escaped, h.escaped())
		})

		t.Run("chunks at a time: "+c.input, func(t *testing.T) {
			var found bytes.Buffer

			input := strings.NewReader(c.input)

			h := &testHandler{}

			filter := NewReader(input, '~', h.handler)

			_, err := io.Copy(&found, filter)
			require.NoError(t, err)

			require.Equal(t, c.expected, found.String())
			require.Equal(t, c.escaped, h.escaped())
		})

		t.Run("1 byte at a time: "+c.input, func(t *testing.T) {
			var found bytes.Buffer

			input := iotest.OneByteReader(strings.NewReader(c.input))

			h := &testHandler{}

			filter := NewReader(input, '~', h.handler)
			_, err := io.Copy(&found, filter)
			require.NoError(t, err)

			require.Equal(t, c.expected, found.String())
			require.Equal(t, c.escaped, h.escaped())
		})

		t.Run("without reading: "+c.input, func(t *testing.T) {
			input := strings.NewReader(c.input)

			h := &testHandler{}

			filter := NewReader(input, '~', h.handler)

			// don't read to mimic a stalled reader
			_ = filter

			assertEventually(t, func() (bool, error) {
				escaped := h.escaped()
				if c.escaped == escaped {
					return true, nil
				}

				return false, fmt.Errorf("expected %v but found %v", c.escaped, escaped)
			})
		})
	}
}

// TestEscapingReader_EmitsPartialReads should emit partial results
// if next character is not read
func TestEscapingReader_FlushesPartialReads(t *testing.T) {
	pr, pw := io.Pipe()

	h := &testHandler{}
	filter := NewReader(pr, '~', h.handler)

	var lock sync.Mutex
	var read bytes.Buffer

	// helper for asserting reads
	requireRead := func(expected *bytes.Buffer) {
		readSoFar := ""

		start := time.Now()
		for time.Since(start) < 2*time.Second {
			lock.Lock()
			readSoFar = read.String()
			lock.Unlock()

			if readSoFar == expected.String() {
				break
			}

			time.Sleep(50 * time.Millisecond)
		}

		require.Equal(t, expected.String(), readSoFar, "timed out without output")
	}

	var rerr error
	var wg sync.WaitGroup
	wg.Add(1)

	// goroutine for reading partial data
	go func() {
		defer wg.Done()

		buf := make([]byte, 1024)
		for {
			n, err := filter.Read(buf)
			lock.Lock()
			read.Write(buf[:n])
			lock.Unlock()

			if err != nil {
				rerr = err
				break
			}
		}
	}()

	expected := &bytes.Buffer{}

	// test basic start and no new lines
	pw.Write([]byte("first data"))
	expected.WriteString("first data")
	requireRead(expected)
	require.Equal(t, "", h.escaped())

	// test ~. appearing in middle of line but stop at new line
	pw.Write([]byte("~.inmiddleappears\n"))
	expected.WriteString("~.inmiddleappears\n")
	requireRead(expected)
	require.Equal(t, "", h.escaped())

	// from here on we test \n~ at boundary

	// ~~ after new line; and stop at \n~
	pw.Write([]byte("~~second line\n~"))
	expected.WriteString("~second line\n")
	requireRead(expected)
	require.Equal(t, "", h.escaped())

	// . to be skipped; stop at \n~ again
	pw.Write([]byte(".third line\n~"))
	expected.WriteString("third line\n")
	requireRead(expected)
	require.Equal(t, ".", h.escaped())

	// q to be emitted; stop at \n
	pw.Write([]byte("qfourth line\n"))
	expected.WriteString("~qfourth line\n")
	requireRead(expected)
	require.Equal(t, ".q", h.escaped())

	// ~. to be skipped; stop at \n~
	pw.Write([]byte("~.fifth line\n~"))
	expected.WriteString("fifth line\n")
	requireRead(expected)
	require.Equal(t, ".q.", h.escaped())

	// ~ alone after \n~ - should be emitted
	pw.Write([]byte("~"))
	expected.WriteString("~")
	requireRead(expected)
	require.Equal(t, ".q.", h.escaped())

	// rest of line ending with \n~
	pw.Write([]byte("rest of line\n~"))
	expected.WriteString("rest of line\n")
	requireRead(expected)
	require.Equal(t, ".q.", h.escaped())

	// m alone after \n~ - should be emitted with ~
	pw.Write([]byte("m"))
	expected.WriteString("~m")
	requireRead(expected)
	require.Equal(t, ".q.m", h.escaped())

	// rest of line and end with \n
	pw.Write([]byte("onemore line\n"))
	expected.WriteString("onemore line\n")
	requireRead(expected)
	require.Equal(t, ".q.m", h.escaped())

	// ~q to be emitted stop at \n~; last charcater
	pw.Write([]byte("~qlast line\n~"))
	expected.WriteString("~qlast line\n")
	requireRead(expected)
	require.Equal(t, ".q.mq", h.escaped())

	// last ~ gets emitted and we preserve error
	eerr := errors.New("my custom error")
	pw.CloseWithError(eerr)
	expected.WriteString("~")
	requireRead(expected)
	require.Equal(t, ".q.mq", h.escaped())

	wg.Wait()
	require.Error(t, rerr)
	require.Equal(t, eerr, rerr)
}

func TestEscapingReader_Generated_EquivalentToNaive(t *testing.T) {
	f := func(v readingInput) bool {
		return checkEquivalenceToNaive(t, string(v))
	}

	require.NoError(t, quick.Check(f, &quick.Config{
		MaxCountScale: 200,
	}))
}

// testHandler is a conveneient struct for finding "escaped" ascii letters
// in escaping reader.
// We avoid complicated unicode characters that may cross byte boundary
type testHandler struct {
	l      sync.Mutex
	result string
}

// handler is method to be passed to escaping io reader
func (t *testHandler) handler(c byte) bool {
	rc := rune(c)
	simple := unicode.IsLetter(rc) ||
		unicode.IsDigit(rc) ||
		unicode.IsPunct(rc) ||
		unicode.IsSymbol(rc)

	if simple {
		t.l.Lock()
		t.result += string([]byte{c})
		t.l.Unlock()
	}
	return c == '.'
}

// escaped returns all seen escaped characters so far
func (t *testHandler) escaped() string {
	t.l.Lock()
	defer t.l.Unlock()

	return t.result
}

// checkEquivalence returns true if parsing input with naive implementation
// is equivalent to our reader
func checkEquivalenceToNaive(t *testing.T, input string) bool {
	nh := &testHandler{}
	expected := naiveEscapeCharacters(input, '~', nh.handler)

	foundH := &testHandler{}

	var inputReader io.Reader = bytes.NewBufferString(input)
	inputReader = &arbtiraryReader{
		buf:         inputReader.(*bytes.Buffer),
		maxReadOnce: 10,
	}
	filter := NewReader(inputReader, '~', foundH.handler)
	var found bytes.Buffer
	_, err := io.Copy(&found, filter)
	if err != nil {
		t.Logf("unexpected error while reading: %v", err)
		return false
	}

	if nh.escaped() == foundH.escaped() && expected == found.String() {
		return true
	}

	t.Logf("escaped differed=%v expected=%v found=%v", nh.escaped() != foundH.escaped(), nh.escaped(), foundH.escaped())
	t.Logf("read  differed=%v expected=%s found=%v", expected != found.String(), expected, found.String())
	return false

}

func TestEscapingReader_Generated_EquivalentToReadOnce(t *testing.T) {
	f := func(v readingInput) bool {
		return checkEquivalenceToNaive(t, string(v))
	}

	require.NoError(t, quick.Check(f, &quick.Config{
		MaxCountScale: 200,
	}))
}

// checkEquivalenceToReadOnce returns true if parsing input in a single
// read matches multiple reads
func checkEquivalenceToReadOnce(t *testing.T, input string) bool {
	nh := &testHandler{}
	var expected bytes.Buffer

	// getting expected value from read all at once
	{
		buf := make([]byte, len(input)+5)
		inputReader := NewReader(bytes.NewBufferString(input), '~', nh.handler)
		_, err := io.CopyBuffer(&expected, inputReader, buf)
		if err != nil {
			t.Logf("unexpected error while reading: %v", err)
			return false
		}
	}

	foundH := &testHandler{}
	var found bytes.Buffer

	// getting found by using arbitrary reader
	{
		inputReader := &arbtiraryReader{
			buf:         bytes.NewBufferString(input),
			maxReadOnce: 10,
		}
		filter := NewReader(inputReader, '~', foundH.handler)
		_, err := io.Copy(&found, filter)
		if err != nil {
			t.Logf("unexpected error while reading: %v", err)
			return false
		}
	}

	if nh.escaped() == foundH.escaped() && expected.String() == found.String() {
		return true
	}

	t.Logf("escaped differed=%v expected=%v found=%v", nh.escaped() != foundH.escaped(), nh.escaped(), foundH.escaped())
	t.Logf("read  differed=%v expected=%s found=%v", expected.String() != found.String(), expected.String(), found.String())
	return false

}

// readingInput is a string with some quick generation capability to
// inject some \n, \n~., \n~q in text
type readingInput string

func (i readingInput) Generate(rand *rand.Rand, size int) reflect.Value {
	v, ok := quick.Value(reflect.TypeOf(""), rand)
	if !ok {
		panic("couldn't generate a string")
	}

	// inject some terminals
	var b bytes.Buffer
	injectProbabilistically := func() {
		p := rand.Float32()
		if p < 0.05 {
			b.WriteString("\n~.")
		} else if p < 0.10 {
			b.WriteString("\n~q")
		} else if p < 0.15 {
			b.WriteString("\n")
		} else if p < 0.2 {
			b.WriteString("~")
		} else if p < 0.25 {
			b.WriteString("~~")
		}
	}

	for _, c := range v.String() {
		injectProbabilistically()
		b.WriteRune(c)
	}

	injectProbabilistically()

	return reflect.ValueOf(readingInput(b.String()))
}

// naiveEscapeCharacters is a simplified implementation that operates
// on entire unchunked string.  Uses regexp implementation.
//
// It differs from the other implementation in handling unicode characters
// proceeding `\n~`
func naiveEscapeCharacters(input string, escapeChar byte, h Handler) string {
	reg := regexp.MustCompile(fmt.Sprintf("(\n|\r)%c.", escapeChar))

	// check first appearances
	if len(input) > 1 && input[0] == escapeChar {
		if input[1] == escapeChar {
			input = input[1:]
		} else if h(input[1]) {
			input = input[2:]
		} else {
			// we are good
		}

	}

	return reg.ReplaceAllStringFunc(input, func(match string) string {
		// match can be more than three bytes because of unicode
		if len(match) < 3 {
			panic(fmt.Errorf("match is less than characters: %d %s", len(match), match))
		}

		c := match[2]

		// ignore some unicode partial codes
		ltr := len(match) > 3 ||
			('a' <= c && c <= 'z') ||
			('A' <= c && c <= 'Z') ||
			('0' <= c && c <= '9') ||
			(c == '~' || c == '.' || c == escapeChar)

		if c == escapeChar {
			return match[:2]
		} else if ltr && h(c) {
			return match[:1]
		} else {
			return match
		}
	})
}

// arbitraryReader is a reader that reads arbitrary length at a time
// to simulate input being read in chunks.
type arbtiraryReader struct {
	buf         *bytes.Buffer
	maxReadOnce int
}

func (r *arbtiraryReader) Read(buf []byte) (int, error) {
	l := r.buf.Len()
	if l == 0 || l == 1 {
		return r.buf.Read(buf)
	}

	if l > r.maxReadOnce {
		l = r.maxReadOnce
	}
	if l != 1 {
		l = rand.Intn(l-1) + 1
	}
	if l > len(buf) {
		l = len(buf)
	}

	return r.buf.Read(buf[:l])
}

func assertEventually(t *testing.T, testFn func() (bool, error)) {
	start := time.Now()
	var err error
	var b bool
	for {
		if time.Since(start) > 2*time.Second {
			assert.Fail(t, "timed out", "error: %v", err)
		}

		b, err = testFn()
		if b {
			return
		}

		time.Sleep(50 * time.Millisecond)
	}
}