/* Copyright 2014 SAP SE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package protocol import ( "fmt" "io" "math" "unicode/utf8" "golang.org/x/text/transform" "github.com/SAP/go-hdb/internal/bufio" "github.com/SAP/go-hdb/internal/unicode" "github.com/SAP/go-hdb/internal/unicode/cesu8" ) const ( locatorIDSize = 8 writeLobRequestHeaderSize = 21 readLobRequestSize = 24 ) // variable (unit testing) //var lobChunkSize = 1 << 14 //TODO: check size var lobChunkSize int32 = 4096 //TODO: check size //lob options type lobOptions int8 const ( loNullindicator lobOptions = 0x01 loDataincluded lobOptions = 0x02 loLastdata lobOptions = 0x04 ) var lobOptionsText = map[lobOptions]string{ loNullindicator: "null indicator", loDataincluded: "data included", loLastdata: "last data", } func (k lobOptions) String() string { t := make([]string, 0, len(lobOptionsText)) for option, text := range lobOptionsText { if (k & option) != 0 { t = append(t, text) } } return fmt.Sprintf("%v", t) } type locatorID uint64 // byte[locatorIdSize] // write lob reply type writeLobReply struct { ids []locatorID numArg int } func (r *writeLobReply) String() string { return fmt.Sprintf("write lob reply: %v", r.ids) } func (r *writeLobReply) kind() partKind { return pkWriteLobReply } func (r *writeLobReply) setNumArg(numArg int) { r.numArg = numArg } func (r *writeLobReply) read(rd *bufio.Reader) error { //resize ids if r.ids == nil || cap(r.ids) < r.numArg { r.ids = make([]locatorID, r.numArg) } else { r.ids = r.ids[:r.numArg] } for i := 0; i < r.numArg; i++ { r.ids[i] = locatorID(rd.ReadUint64()) } return rd.GetError() } //write lob request type writeLobRequest struct { lobPrmFields []*ParameterField } func (r *writeLobRequest) kind() partKind { return pkWriteLobRequest } func (r *writeLobRequest) size() (int, error) { // TODO: check size limit size := 0 for _, prmField := range r.lobPrmFields { cr := prmField.chunkReader if cr.done() { continue } if err := cr.fill(); err != nil { return 0, err } size += writeLobRequestHeaderSize size += cr.size() } return size, nil } func (r *writeLobRequest) numArg() int { n := 0 for _, prmField := range r.lobPrmFields { cr := prmField.chunkReader if !cr.done() { n++ } } return n } func (r *writeLobRequest) write(wr *bufio.Writer) error { for _, prmField := range r.lobPrmFields { cr := prmField.chunkReader if !cr.done() { wr.WriteUint64(uint64(prmField.lobLocatorID)) opt := int8(0x02) // data included if cr.eof() { opt |= 0x04 // last data } wr.WriteInt8(opt) wr.WriteInt64(-1) //offset (-1 := append) wr.WriteInt32(int32(cr.size())) // size wr.Write(cr.bytes()) } } return nil } //read lob request type readLobRequest struct { w lobChunkWriter } func (r *readLobRequest) kind() partKind { return pkReadLobRequest } func (r *readLobRequest) size() (int, error) { return readLobRequestSize, nil } func (r *readLobRequest) numArg() int { return 1 } func (r *readLobRequest) write(wr *bufio.Writer) error { wr.WriteUint64(uint64(r.w.id())) readOfs, readLen := r.w.readOfsLen() wr.WriteInt64(readOfs + 1) //1-based wr.WriteInt32(readLen) wr.WriteZeroes(4) return nil } // read lob reply // - seems like readLobreply gives only an result for one lob - even if more then one is requested // --> read single lobs type readLobReply struct { w lobChunkWriter } func (r *readLobReply) kind() partKind { return pkReadLobReply } func (r *readLobReply) setNumArg(numArg int) { if numArg != 1 { panic("numArg == 1 expected") } } func (r *readLobReply) read(rd *bufio.Reader) error { id := rd.ReadUint64() if r.w.id() != locatorID(id) { return fmt.Errorf("internal error: invalid lob locator %d - expected %d", id, r.w.id()) } opt := rd.ReadInt8() chunkLen := rd.ReadInt32() rd.Skip(3) eof := (lobOptions(opt) & loLastdata) != 0 if err := r.w.write(rd, int(chunkLen), eof); err != nil { return err } return rd.GetError() } // lobChunkReader reads lob field io.Reader in chunks for writing to db. type lobChunkReader interface { fill() error size() int bytes() []byte eof() bool done() bool } func newLobChunkReader(isCharBased bool, r io.Reader) lobChunkReader { if isCharBased { return &charLobChunkReader{r: r} } return &binaryLobChunkReader{r: r} } // binaryLobChunkReader (byte based chunks). type binaryLobChunkReader struct { r io.Reader _size int _eof bool _done bool b []byte } func (l *binaryLobChunkReader) eof() bool { return l._eof } func (l *binaryLobChunkReader) done() bool { return l._done } func (l *binaryLobChunkReader) size() int { return l._size } func (l *binaryLobChunkReader) bytes() []byte { l._done = l._eof return l.b[:l._size] } func (l *binaryLobChunkReader) fill() error { if l._eof { return io.EOF } var err error l.b = resizeBuffer(l.b, int(lobChunkSize)) l._size, err = l.r.Read(l.b) if err != nil && err != io.EOF { return err } l._eof = err == io.EOF return nil } // charLobChunkReader (cesu8 character based chunks). type charLobChunkReader struct { r io.Reader _size int _eof bool _done bool b []byte c []byte ofs int } func (l *charLobChunkReader) eof() bool { return l._eof } func (l *charLobChunkReader) done() bool { return l._done } func (l *charLobChunkReader) size() int { return l._size } func (l *charLobChunkReader) bytes() []byte { l._done = l._eof return l.b[:l._size] } func (l *charLobChunkReader) fill() error { if l._eof { return io.EOF } l.c = resizeBuffer(l.c, int(lobChunkSize)+l.ofs) n, err := l.r.Read(l.c[l.ofs:]) size := n + l.ofs if err != nil && err != io.EOF { return err } l._eof = err == io.EOF if l._eof && size == 0 { l._size = 0 return nil } l.b = resizeBuffer(l.b, cesu8.Size(l.c[:size])) // last rune might be incomplete, so size is one greater than needed nDst, nSrc, err := unicode.Utf8ToCesu8Transformer.Transform(l.b, l.c[:size], l._eof) if err != nil && err != transform.ErrShortSrc { return err } if l._eof && err == transform.ErrShortSrc { return unicode.ErrInvalidUtf8 } l._size = nDst l.ofs = size - nSrc if l.ofs > 0 { copy(l.c, l.c[nSrc:size]) // copy rest to buffer beginn } return nil } // lobChunkWriter reads db lob chunks and writes them into lob field io.Writer. type lobChunkWriter interface { SetWriter(w io.Writer) error // gets called by driver.Lob.Scan id() locatorID write(rd *bufio.Reader, size int, eof bool) error readOfsLen() (int64, int32) eof() bool } func newLobChunkWriter(isCharBased bool, s *Session, id locatorID, charLen, byteLen int64) lobChunkWriter { if isCharBased { return &charLobChunkWriter{s: s, _id: id, charLen: charLen, byteLen: byteLen} } return &binaryLobChunkWriter{s: s, _id: id, charLen: charLen, byteLen: byteLen} } // binaryLobChunkWriter (byte based lobs). type binaryLobChunkWriter struct { s *Session _id locatorID charLen int64 byteLen int64 readOfs int64 _eof bool ofs int wr io.Writer b []byte } func (l *binaryLobChunkWriter) id() locatorID { return l._id } func (l *binaryLobChunkWriter) eof() bool { return l._eof } func (l *binaryLobChunkWriter) SetWriter(wr io.Writer) error { l.wr = wr if err := l.flush(); err != nil { return err } return l.s.readLobStream(l) } func (l *binaryLobChunkWriter) write(rd *bufio.Reader, size int, eof bool) error { l._eof = eof // store eof if size == 0 { return nil } l.b = resizeBuffer(l.b, size+l.ofs) rd.ReadFull(l.b[l.ofs:]) if l.wr != nil { return l.flush() } return nil } func (l *binaryLobChunkWriter) readOfsLen() (int64, int32) { readLen := l.charLen - l.readOfs if readLen > int64(math.MaxInt32) || readLen > int64(lobChunkSize) { return l.readOfs, lobChunkSize } return l.readOfs, int32(readLen) } func (l *binaryLobChunkWriter) flush() error { if _, err := l.wr.Write(l.b); err != nil { return err } l.readOfs += int64(len(l.b)) return nil } type charLobChunkWriter struct { s *Session _id locatorID charLen int64 byteLen int64 readOfs int64 _eof bool ofs int wr io.Writer b []byte } func (l *charLobChunkWriter) id() locatorID { return l._id } func (l *charLobChunkWriter) eof() bool { return l._eof } func (l *charLobChunkWriter) SetWriter(wr io.Writer) error { l.wr = wr if err := l.flush(); err != nil { return err } return l.s.readLobStream(l) } func (l *charLobChunkWriter) write(rd *bufio.Reader, size int, eof bool) error { l._eof = eof // store eof if size == 0 { return nil } l.b = resizeBuffer(l.b, size+l.ofs) rd.ReadFull(l.b[l.ofs:]) if l.wr != nil { return l.flush() } return nil } func (l *charLobChunkWriter) readOfsLen() (int64, int32) { readLen := l.charLen - l.readOfs if readLen > int64(math.MaxInt32) || readLen > int64(lobChunkSize) { return l.readOfs, lobChunkSize } return l.readOfs, int32(readLen) } func (l *charLobChunkWriter) flush() error { nDst, nSrc, err := unicode.Cesu8ToUtf8Transformer.Transform(l.b, l.b, true) // inline cesu8 to utf8 transformation if err != nil && err != transform.ErrShortSrc { return err } if _, err := l.wr.Write(l.b[:nDst]); err != nil { return err } l.ofs = len(l.b) - nSrc if l.ofs != 0 && l.ofs != cesu8.CESUMax/2 { // assert remaining bytes return unicode.ErrInvalidCesu8 } l.readOfs += int64(l.runeCount(l.b[:nDst])) if l.ofs != 0 { l.readOfs++ // add half encoding copy(l.b, l.b[nSrc:len(l.b)]) // move half encoding to buffer begin } return nil } // Caution: hdb counts 4 byte utf-8 encodings (cesu-8 6 bytes) as 2 (3 byte) chars func (l *charLobChunkWriter) runeCount(b []byte) int { numChars := 0 for len(b) > 0 { _, size := utf8.DecodeRune(b) b = b[size:] numChars++ if size == utf8.UTFMax { numChars++ } } return numChars } // helper func resizeBuffer(b1 []byte, size int) []byte { if b1 == nil || cap(b1) < size { b2 := make([]byte, size) copy(b2, b1) // !!! return b2 } return b1[:size] }