Skip to content

Commit edfd5c9

Browse files
Use Unicode 15 tables for unicode normalization and segmentation
To match with the Unicode support in Go 1.21, we'll now use the Unicode 15 tables when we're normalizing Unicode strings and when counting user-perceived characters ("grapheme clusters") for source position purposes.
1 parent fef0406 commit edfd5c9

File tree

8 files changed

+16
-13
lines changed

8 files changed

+16
-13
lines changed

go.mod

+4-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.18
55
require (
66
github.com/agext/levenshtein v1.2.1
77
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3
8-
github.com/apparentlymart/go-textseg/v13 v13.0.0
8+
github.com/apparentlymart/go-textseg/v15 v15.0.0
99
github.com/davecgh/go-spew v1.1.1
1010
github.com/go-test/deep v1.0.3
1111
github.com/google/go-cmp v0.3.1
@@ -20,10 +20,11 @@ require (
2020
)
2121

2222
require (
23+
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
2324
github.com/kr/text v0.1.0 // indirect
2425
github.com/pmezard/go-difflib v1.0.0 // indirect
2526
github.com/stretchr/testify v1.2.2 // indirect
26-
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
27+
golang.org/x/sys v0.5.0 // indirect
2728
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
28-
golang.org/x/text v0.3.8 // indirect
29+
golang.org/x/text v0.11.0 // indirect
2930
)

go.sum

+6-4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:o
55
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
66
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
77
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
8+
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
9+
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
810
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
911
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1012
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
@@ -40,12 +42,12 @@ golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0
4042
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
4143
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
4244
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
43-
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
44-
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
45+
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
46+
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
4547
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
4648
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
4749
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
48-
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
49-
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
50+
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
51+
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
5052
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
5153
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

hclsyntax/parser.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"strconv"
1010
"unicode/utf8"
1111

12-
"github.com/apparentlymart/go-textseg/v13/textseg"
12+
"github.com/apparentlymart/go-textseg/v15/textseg"
1313
"github.com/hashicorp/hcl/v2"
1414
"github.com/zclconf/go-cty/cty"
1515
)

hclsyntax/parser_template.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"strings"
99
"unicode"
1010

11-
"github.com/apparentlymart/go-textseg/v13/textseg"
11+
"github.com/apparentlymart/go-textseg/v15/textseg"
1212
"github.com/hashicorp/hcl/v2"
1313
"github.com/zclconf/go-cty/cty"
1414
)

hclsyntax/token.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"bytes"
88
"fmt"
99

10-
"github.com/apparentlymart/go-textseg/v13/textseg"
10+
"github.com/apparentlymart/go-textseg/v15/textseg"
1111
"github.com/hashicorp/hcl/v2"
1212
)
1313

hclwrite/tokens.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"bytes"
88
"io"
99

10-
"github.com/apparentlymart/go-textseg/v13/textseg"
10+
"github.com/apparentlymart/go-textseg/v15/textseg"
1111
"github.com/hashicorp/hcl/v2"
1212
"github.com/hashicorp/hcl/v2/hclsyntax"
1313
)

json/scanner.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ package json
66
import (
77
"fmt"
88

9-
"github.com/apparentlymart/go-textseg/v13/textseg"
9+
"github.com/apparentlymart/go-textseg/v15/textseg"
1010
"github.com/hashicorp/hcl/v2"
1111
)
1212

pos_scanner.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"bufio"
88
"bytes"
99

10-
"github.com/apparentlymart/go-textseg/v13/textseg"
10+
"github.com/apparentlymart/go-textseg/v15/textseg"
1111
)
1212

1313
// RangeScanner is a helper that will scan over a buffer using a bufio.SplitFunc

0 commit comments

Comments
 (0)