Skip to content

Encode crash #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
coanor wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Encode crash #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6ed3588
save
coanor May 23, 2024
e66ed3e
use cap to detect encode buffer size
coanor May 23, 2024
bde7a8b
update point check warnning
coanor May 24, 2024
6b3f464
add new error when protobuf encoding index out of range
coanor May 27, 2024
b99b311
recover proto generated file
coanor Jun 1, 2024
bac55ba
save
coanor Jun 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 21 additions & 21 deletions point/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ func (c *checker) checkMeasurement(m string) string {

if c.cfg.maxMeasurementLen > 0 && len(m) > c.cfg.maxMeasurementLen {
c.addWarn(WarnInvalidMeasurement,
fmt.Sprintf("exceed max measurement length(%d), got length %d, trimmed",
c.cfg.maxMeasurementLen, len(m)))
fmt.Sprintf("%s: exceed max measurement length(%d), got length %d, trimmed",
m, c.cfg.maxMeasurementLen, len(m)))
return m[:c.cfg.maxMeasurementLen]
} else {
return m
Expand Down Expand Up @@ -151,16 +151,16 @@ func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {

if c.cfg.maxTagValLen > 0 && len(x.S) > c.cfg.maxTagValLen {
c.addWarn(WarnMaxTagValueLen,
fmt.Sprintf("exceed max tag value length(%d), got %d, value truncated",
c.cfg.maxTagValLen, len(x.S)))
fmt.Sprintf("%s: exceed max tag value length(%d), got %d, value truncated",
x, c.cfg.maxTagValLen, len(x.S)))

x.S = x.S[:c.cfg.maxTagValLen]
f.Val = x
}

// check tag key '\', '\n'
if strings.HasSuffix(f.Key, `\`) || strings.Contains(f.Key, "\n") {
c.addWarn(WarnInvalidTagKey, fmt.Sprintf("invalid tag key `%s'", f.Key))
c.addWarn(WarnInvalidTagKey, fmt.Sprintf("%s: invalid tag key `%s'", f, f.Key))

newKey := adjustKV(f.Key)
if c.keyConflict(newKey, kvs) {
Expand All @@ -172,15 +172,15 @@ func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {

// check tag value: '\', '\n'
if strings.HasSuffix(f.GetS(), `\`) || strings.Contains(f.GetS(), "\n") {
c.addWarn(WarnInvalidTagValue, fmt.Sprintf("invalid tag value %q", f.GetS()))
c.addWarn(WarnInvalidTagValue, fmt.Sprintf("%s: invalid tag value %q", f, f.GetS()))

x.S = adjustKV(x.S)
f.Val = x
}

// replace `.' with `_' in tag keys
if strings.Contains(f.Key, ".") && !c.cfg.enableDotInKey {
c.addWarn(WarnInvalidTagKey, fmt.Sprintf("invalid tag key `%s': found `.'", f.Key))
c.addWarn(WarnInvalidTagKey, fmt.Sprintf("%s: invalid tag key `%s': found `.'", f, f.Key))

newKey := strings.ReplaceAll(f.Key, ".", "_")
if c.keyConflict(newKey, kvs) {
Expand All @@ -191,7 +191,7 @@ func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {
}

if c.keyDisabled(f.Key) {
c.addWarn(WarnTagDisabled, fmt.Sprintf("tag key `%s' disabled", f.Key))
c.addWarn(WarnTagDisabled, fmt.Sprintf("%s: tag key `%s' disabled", f, f.Key))
return f, false
}

Expand All @@ -204,8 +204,8 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {
// trim key
if c.cfg.maxFieldKeyLen > 0 && len(f.Key) > c.cfg.maxFieldKeyLen {
c.addWarn(WarnMaxFieldKeyLen,
fmt.Sprintf("exceed max field key length(%d), got %d, key truncated to %s",
c.cfg.maxFieldKeyLen, len(f.Key), f.Key))
fmt.Sprintf("%s: exceed max field key length(%d), got %d, key truncated to %s",
f, c.cfg.maxFieldKeyLen, len(f.Key), f.Key))

newKey := f.Key[:c.cfg.maxFieldKeyLen]

Expand All @@ -218,7 +218,7 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {

if strings.Contains(f.Key, ".") && !c.cfg.enableDotInKey {
c.addWarn(WarnDotInkey,
fmt.Sprintf("invalid field key `%s': found `.'", f.Key))
fmt.Sprintf("%s: invalid field key `%s': found `.'", f, f.Key))

newKey := strings.ReplaceAll(f.Key, ".", "_")
if c.keyConflict(newKey, kvs) {
Expand All @@ -230,7 +230,7 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {

if c.keyDisabled(f.Key) {
c.addWarn(WarnFieldDisabled,
fmt.Sprintf("field key `%s' disabled, value: %v", f.Key, f.Raw()))
fmt.Sprintf("%s: field key `%s' disabled, value: %v", f, f.Key, f.Raw()))
return nil, false
}

Expand All @@ -239,7 +239,8 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {
if !c.cfg.enableU64Field {
if x.U > uint64(math.MaxInt64) {
c.addWarn(WarnMaxFieldValueInt,
fmt.Sprintf("too large int field: key=%s, value=%d(> %d)", f.Key, x.U, uint64(math.MaxInt64)))
fmt.Sprintf("%s: too large int field: key=%s, value=%d(> %d)",
f, f.Key, x.U, uint64(math.MaxInt64)))
return f, false
} else {
// Force convert uint64 to int64: to disable line proto like
Expand All @@ -265,14 +266,14 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {

if !c.cfg.enableStrField {
c.addWarn(WarnInvalidFieldValueType,
fmt.Sprintf("field(%s) dropped with string value, when [DisableStringField] enabled", f.Key))
fmt.Sprintf("%s: field(%s) dropped with string value, when [DisableStringField] enabled", f, f.Key))
return f, false
}

if c.cfg.maxFieldValLen > 0 && len(x.D) > c.cfg.maxFieldValLen {
c.addWarn(WarnMaxFieldValueLen,
fmt.Sprintf("field (%s) exceed max field value length(%d), got %d, value truncated",
f.Key, c.cfg.maxFieldValLen, len(x.D)))
fmt.Sprintf("%s: field (%s) exceed max field value length(%d), got %d, value truncated",
f, f.Key, c.cfg.maxFieldValLen, len(x.D)))

x.D = x.D[:c.cfg.maxFieldValLen]
f.Val = x
Expand All @@ -282,23 +283,22 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {

if !c.cfg.enableStrField {
c.addWarn(WarnInvalidFieldValueType,
fmt.Sprintf("field(%s) dropped with string value, when [DisableStringField] enabled", f.Key))
fmt.Sprintf("%s: field(%s) dropped with string value, when [DisableStringField] enabled", f, f.Key))
return f, false
}

if c.cfg.maxFieldValLen > 0 && len(x.S) > c.cfg.maxFieldValLen {
c.addWarn(WarnMaxFieldValueLen,
fmt.Sprintf("field (%s) exceed max field value length(%d), got %d, value truncated",
f.Key, c.cfg.maxFieldValLen, len(x.S)))
fmt.Sprintf("%s: field (%s) exceed max field value length(%d), got %d, value truncated",
f, f.Key, c.cfg.maxFieldValLen, len(x.S)))

x.S = x.S[:c.cfg.maxFieldValLen]
f.Val = x
}

default:
c.addWarn(WarnInvalidFieldValueType,
fmt.Sprintf("invalid field (%s), value: %s, type: %s",
f.Key, f.Val, reflect.TypeOf(f.Val)))
fmt.Sprintf("%s: invalid field (%s), value: %s, type: %s", f, f.Key, f.Val, reflect.TypeOf(f.Val)))
return f, false
}

Expand Down
4 changes: 2 additions & 2 deletions point/encode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,14 +317,14 @@ func (e *Encoder) doEncodeProtobuf(buf []byte) ([]byte, bool) {

// e.pbpts size larger than buf, we must trim some of points
// until size fit ok or MarshalTo will panic.
if curSize >= len(buf) {
if curSize >= cap(buf) {
if len(e.pbpts.Arr) <= 1 { // nothing to trim
e.lastErr = errTooSmallBuffer
return nil, false
}

for {
if pbptsSize = e.pbpts.Size(); pbptsSize > len(buf) {
if pbptsSize = e.pbpts.Size(); pbptsSize > cap(buf) {
e.pbpts.Arr = e.pbpts.Arr[:len(e.pbpts.Arr)-trimmed]
e.lastPtsIdx -= trimmed
trimmed *= 2
Expand Down
2 changes: 1 addition & 1 deletion point/encode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ func TestV2Encode(t *T.T) {
var (
decodePts []*Point
round int
buf = make([]byte, 1<<20) // KB
buf = make([]byte, 0, 1<<20) // KB
)

for {
Expand Down
11 changes: 1 addition & 10 deletions point/point.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,16 +466,7 @@ func (p *Point) LPSize() int {

// PBSize get point protobuf size.
func (p *Point) PBSize() int {
pbpt := p.PBPoint()

m := protojson.Marshaler{}
buf := bytes.Buffer{}

if err := m.Marshal(&buf, pbpt); err != nil {
return 0
}

return buf.Len()
return p.pt.Size()
}

func b64(x []byte) string {
Expand Down
Loading