replace interface{} to any, add fuzz test and fi… …x bug -0 (#48)

* fix & style: fix the issue that datax output too much log, replace the package "io/ioutil". * style: replace the package "io/ioutil" * style & test & fix : replace interface{} to any, add fuzz test and fix bug -0 * style: add license
Breeze0806 · Aug 16, 2024 · 5e171af · 5e171af
1 parent ec574a9
commit 5e171af
Show file tree

Hide file tree

Showing 53 changed files with 200 additions and 185 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,6 @@ release/*
 *.tar.gz
 *.chn
 *.en
-
+testdata
 *.swp
 tags
diff --git a/datax/README.md b/datax/README.md
@@ -152,7 +152,7 @@ type Querier interface {
  // Checks connectivity
  PingContext(ctx context.Context) error
  // Queries using a query statement
- QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+ QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
  // Obtains a specific table based on parameters
  FetchTableWithParam(ctx context.Context, param database.Parameter) (database.Table, error)
  // Retrieves records using parameters and a handler
@@ -278,9 +278,9 @@ type Execer interface {
  // Checks connectivity
  PingContext(ctx context.Context) error
  // Queries using a query statement
- QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+ QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
  // Executes a query statement
- ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+ ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
  // Obtains a specific table based on parameters
  FetchTableWithParam(ctx context.Context, param database.Parameter) (database.Table, error)
  // Performs batch execution

diff --git a/datax/README_zh-CN.md b/datax/README_zh-CN.md
@@ -158,7 +158,7 @@ type Querier interface {
 	//检测连通性
 	PingContext(ctx context.Context) error
 	//通过query查询语句进行查询
-	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
 	//通过参数param获取具体表
 	FetchTableWithParam(ctx context.Context, param database.Parameter) (database.Table, error)
 	//通过参数param，处理句柄handler获取记录
@@ -284,9 +284,9 @@ type Execer interface {
 	//检测连通性
 	PingContext(ctx context.Context) error
 	//通过query查询语句进行查询
-	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
 	//通过query查询语句进行查询
-	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
 	//通过参数param获取具体表
 	FetchTableWithParam(ctx context.Context, param database.Parameter) (database.Table, error)
 	//批量执行

diff --git a/datax/common/plugin/task.go b/datax/common/plugin/task.go
@@ -42,7 +42,7 @@ type Task interface {
 	// Set Task ID
 	SetTaskID(taskID int64)
 	// Wrap Error
-	Wrapf(err error, format string, args ...interface{}) error
+	Wrapf(err error, format string, args ...any) error
 	// Format - Log format
 	Format(format string) string
 }
@@ -105,7 +105,7 @@ func (b *BaseTask) SetJobID(jobID int64) {
 }
 
 // Wrapf - Wraps an error with additional context
-func (b *BaseTask) Wrapf(err error, format string, args ...interface{}) error {
+func (b *BaseTask) Wrapf(err error, format string, args ...any) error {
 	return errors.Wrapf(err, b.Format(format), args...)
 }
 

diff --git a/datax/common/plugin/task_test.go b/datax/common/plugin/task_test.go
@@ -170,7 +170,7 @@ func TestBaseTask_Wrapf(t *testing.T) {
 	type args struct {
 		err    error
 		format string
-		args   []interface{}
+		args   []any
 	}
 	tests := []struct {
 		name    string

diff --git a/datax/core/job/help_test.go b/datax/core/job/help_test.go
@@ -225,7 +225,7 @@ func (m *mockWriter) Task() writer.Task {
 }
 
 func equalConfigJSON(gotConfig, wantConfig *config.JSON) bool {
-	var got, want interface{}
+	var got, want any
 	err := json.Unmarshal([]byte(gotConfig.String()), &got)
 	if err != nil {
 		panic(err)

diff --git a/datax/core/statistics/container/metrics.go b/datax/core/statistics/container/metrics.go
@@ -43,7 +43,7 @@ func (m *Metrics) JSON() *encoding.JSON {
 }
 
 // Set - Sets the value at the specified path
-func (m *Metrics) Set(path string, value interface{}) error {
+func (m *Metrics) Set(path string, value any) error {
 	m.Lock()
 	defer m.Unlock()
 	return m.metricJSON.Set(path, value)

diff --git a/datax/core/statistics/container/metrics_test.go b/datax/core/statistics/container/metrics_test.go
@@ -73,7 +73,7 @@ func TestNewMetrics(t *testing.T) {
 func TestMetrics_Set(t *testing.T) {
 	type args struct {
 		path  string
-		value interface{}
+		value any
 	}
 	tests := []struct {
 		name    string

diff --git a/datax/plugin/reader/dbms/parameter.go b/datax/plugin/reader/dbms/parameter.go
@@ -69,7 +69,7 @@ func (t *TableParam) Query(_ []element.Record) (string, error) {
 }
 
 // Agrs Get query parameters
-func (t *TableParam) Agrs(_ []element.Record) ([]interface{}, error) {
+func (t *TableParam) Agrs(_ []element.Record) ([]any, error) {
 	return nil, nil
 }
 
@@ -119,7 +119,7 @@ func (q *QueryParam) Query(_ []element.Record) (string, error) {
 }
 
 // Agrs Get query parameters
-func (q *QueryParam) Agrs(_ []element.Record) (a []interface{}, err error) {
+func (q *QueryParam) Agrs(_ []element.Record) (a []any, err error) {
 	if len(q.Config.GetQuerySQL()) > 0 {
 		return nil, nil
 	}
@@ -134,7 +134,7 @@ func (q *QueryParam) Agrs(_ []element.Record) (a []interface{}, err error) {
 				if right, err = q.Config.GetSplitConfig().Range.rightColumn(v.Name()); err != nil {
 					return
 				}
-				var li, ri interface{}
+				var li, ri any
 				if li, err = v.Valuer(left).Value(); err != nil {
 					return
 				}
@@ -178,7 +178,7 @@ func (s *SplitParam) Query(_ []element.Record) (string, error) {
 }
 
 // Agrs Get query parameters
-func (s *SplitParam) Agrs(_ []element.Record) ([]interface{}, error) {
+func (s *SplitParam) Agrs(_ []element.Record) ([]any, error) {
 	return nil, nil
 }
 
@@ -213,7 +213,7 @@ func (m *MinParam) Query(_ []element.Record) (string, error) {
 }
 
 // Agrs Get query parameters
-func (m *MinParam) Agrs(_ []element.Record) ([]interface{}, error) {
+func (m *MinParam) Agrs(_ []element.Record) ([]any, error) {
 	return nil, nil
 }
 
@@ -248,6 +248,6 @@ func (m *MaxParam) Query(_ []element.Record) (string, error) {
 }
 
 // Agrs Get query parameters
-func (m *MaxParam) Agrs(_ []element.Record) ([]interface{}, error) {
+func (m *MaxParam) Agrs(_ []element.Record) ([]any, error) {
 	return nil, nil
 }
diff --git a/datax/plugin/reader/dbms/parameter_test.go b/datax/plugin/reader/dbms/parameter_test.go
@@ -121,7 +121,7 @@ func Test_tableParam_Agrs(t *testing.T) {
 		name    string
 		t       *TableParam
 		args    args
-		want    []interface{}
+		want    []any
 		wantErr bool
 	}{
 		{
@@ -263,7 +263,7 @@ func Test_queryParam_Agrs(t *testing.T) {
 		q       *QueryParam
 		args    args
 		config  *BaseConfig
-		want    []interface{}
+		want    []any
 		wantErr bool
 	}{
 		{
@@ -295,7 +295,7 @@ func Test_queryParam_Agrs(t *testing.T) {
 			args: args{
 				in0: nil,
 			},
-			want: []interface{}{
+			want: []any{
 				int64(11), int64(22),
 			},
 		},
@@ -472,7 +472,7 @@ func TestSplitParam_Agrs(t *testing.T) {
 		name    string
 		s       *SplitParam
 		args    args
-		want    []interface{}
+		want    []any
 		wantErr bool
 	}{
 		{
@@ -567,7 +567,7 @@ func TestMinParam_Agrs(t *testing.T) {
 		name    string
 		m       *MinParam
 		args    args
-		want    []interface{}
+		want    []any
 		wantErr bool
 	}{
 		{
@@ -652,7 +652,7 @@ func TestMaxParam_Agrs(t *testing.T) {
 		name    string
 		m       *MaxParam
 		args    args
-		want    []interface{}
+		want    []any
 		wantErr bool
 	}{
 		{

diff --git a/datax/plugin/reader/dbms/querier.go b/datax/plugin/reader/dbms/querier.go
@@ -28,7 +28,7 @@ type Querier interface {
 	// Check connectivity.
 	PingContext(ctx context.Context) error
 	// Perform a query using the specified query statement.
-	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
 	// Obtain a specific table based on the provided parameters.
 	FetchTableWithParam(ctx context.Context, param database.Parameter) (database.Table, error)
 	// Retrieve records using the provided parameters and the handler.

diff --git a/datax/plugin/reader/dbms/querier_help_test.go b/datax/plugin/reader/dbms/querier_help_test.go
@@ -121,7 +121,7 @@ func (m *MockQuerier) PingContext(ctx context.Context) error {
 	return m.PingErr
 }
 
-func (m *MockQuerier) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
+func (m *MockQuerier) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
 	return nil, m.QueryErr
 }
 
@@ -216,7 +216,7 @@ func (m *MockSender) Shutdown() error {
 }
 
 func equalConfigJSON(gotConfig, wantConfig *config.JSON) bool {
-	var got, want interface{}
+	var got, want any
 	err := json.Unmarshal([]byte(gotConfig.String()), &got)
 	if err != nil {
 		panic(err)

diff --git a/datax/plugin/writer/dbms/execer.go b/datax/plugin/writer/dbms/execer.go
@@ -26,9 +26,9 @@ type Execer interface {
 	// Obtain relational database configuration through configuration
 	PingContext(ctx context.Context) error
 
-	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
 	// BaseDbHandler Basic Database Execution Handler Encapsulation
-	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
 
 	FetchTableWithParam(ctx context.Context, param database.Parameter) (database.Table, error)
 

diff --git a/datax/plugin/writer/dbms/execer_help_test.go b/datax/plugin/writer/dbms/execer_help_test.go
@@ -150,11 +150,11 @@ func (m *MockExecer) PingContext(ctx context.Context) error {
 	return m.PingErr
 }
 
-func (m *MockExecer) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
+func (m *MockExecer) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
 	return nil, m.QueryErr
 }
 
-func (m *MockExecer) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
+func (m *MockExecer) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
 	if query == "wait" {
 		time.Sleep(100 * time.Millisecond)
 	}
@@ -245,7 +245,7 @@ func (m *MockReceiver) Shutdown() error {
 }
 
 func equalConfigJSON(gotConfig, wantConfig *config.JSON) bool {
-	var got, want interface{}
+	var got, want any
 	err := json.Unmarshal([]byte(gotConfig.String()), &got)
 	if err != nil {
 		panic(err)

diff --git a/element/column.go b/element/column.go
@@ -207,7 +207,7 @@ func (d *DefaultColumn) AsFloat64() (float64, error) {
 }
 
 // ByteSize Byte size
-func ByteSize(src interface{}) int {
+func ByteSize(src any) int {
 	switch data := src.(type) {
 	case nil:
 		return 0

diff --git a/element/column_test.go b/element/column_test.go
@@ -527,7 +527,7 @@ func TestDefaultColumn_Cmp(t *testing.T) {
 
 func TestByteSize(t *testing.T) {
 	type args struct {
-		src interface{}
+		src any
 	}
 	tests := []struct {
 		name string

diff --git a/element/error.go b/element/error.go
@@ -77,7 +77,7 @@ type SetError struct {
 }
 
 // NewSetError Generates a setting error by setting the value i to the specified other type with the error err
-func NewSetError(i interface{}, other ColumnType, err error) *SetError {
+func NewSetError(i any, other ColumnType, err error) *SetError {
 	for uerr := err; uerr != nil; uerr = errors.Unwrap(err) {
 		err = uerr
 	}

diff --git a/element/help_test.go b/element/help_test.go
@@ -26,7 +26,7 @@ type mockTimeDecoder struct {
 	StringTimeDecoder
 }
 
-func (m *mockTimeDecoder) TimeDecode(t time.Time) (interface{}, error) {
+func (m *mockTimeDecoder) TimeDecode(t time.Time) (any, error) {
 	return time.Time{}, fmt.Errorf("mockTimeDecoder error")
 }
 

diff --git a/element/number.go b/element/number.go
@@ -606,6 +606,9 @@ func (c *Converter) ConvertBigInt(s string) (num BigIntNumber, err error) {
 	if len(first) == 0 {
 		first = "0"
 	}
+	if first == "0" {
+		sign = ""
+	}
 	return convertBigInt(sign + first).(BigIntNumber), nil
 }
 
@@ -685,6 +688,9 @@ func convertDecimal(s string) (num DecimalNumber, err error) {
 			first = "0"
 		}
 		if len(s[pIndex+1:]) == 0 {
+			if first == "0" {
+				sign = ""
+			}
 			return &DecimalStr{
 				value:  sign + first,
 				intLen: len(sign) + len(first),
@@ -705,6 +711,7 @@ func convertDecimal(s string) (num DecimalNumber, err error) {
 	if len(first) == 0 {
 		first = "0"
 	}
+
 	return convertBigInt(sign + first).(DecimalNumber), nil
 }
 

diff --git a/element/number_fuzz_test.go b/element/number_fuzz_test.go
@@ -0,0 +1,44 @@
+// Copyright 2020 the go-etl Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build go1.18
+// +build go1.18
+
+package element
+
+import (
+	"testing"
+)
+
+func FuzzConverterConvertDecimal(f *testing.F) {
+	for _, t := range testTableDecimalStr {
+		f.Add(t.short)
+	}
+
+	f.Fuzz(func(t *testing.T, number string) {
+		num1, err1 := testNumConverter.ConvertDecimal(number)
+		num2, err2 := testOldNumConverter.ConvertDecimal(number)
+		if err1 == nil && err2 != nil {
+			t.Fatalf("input: %v err1: %v err2: %v", number, err1, err2)
+		}
+		if err1 != nil && err2 == nil {
+			t.Fatalf("input: %v err1: %v err2:%v", number, err1, err2)
+		}
+		if err1 == nil && err2 == nil {
+			if num1.String() != num2.String() {
+				t.Fatalf("input: %v num1: %v num2: %v", number, num1.String(), num2.String())
+			}
+		}
+	})
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -19,6 +19,6 @@ release/* @@
     *.tar.gz
     *.chn
     *.en
+    testdata
     *.swp
     tags