Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Updates to build with the latest SDK #179

Merged
merged 9 commits into from
Feb 20, 2019
Merged

Updates to build with the latest SDK #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a92fe3f
gitignore
Feb 13, 2019
8db3e74
Checkpoint with the C# like transform
Feb 14, 2019
0af4290
Changes for the latest SDK
Feb 14, 2019
08af62a
Update driver/normalizer/normalizer.go
dennwc Feb 14, 2019
b74998a
Update driver/normalizer/normalizer.go
dennwc Feb 14, 2019
cf6afae
Move the function decorators to the same object as async in the
Feb 15, 2019
6fcd35d
Fixes
Feb 15, 2019
abf0e5a
Add comment to explain that a nil or missing return field doesnt mean…
Feb 15, 2019
e884740
Update driver/normalizer/normalizer.go
bzz Feb 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ fixtures/*_got
vendor/*
__pycache__
makebuild.sh
vendor/*
6 changes: 3 additions & 3 deletions Gopkg.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Gopkg.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

[[constraint]]
name = "gopkg.in/bblfsh/sdk.v2"
version = "2.13.4"
version = "2.14.x"

[prune]
go-tests = true
Expand Down
1 change: 1 addition & 0 deletions driver/fixtures/fixtures_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ var Suite = &fixtures.Suite{
Semantic: fixtures.SemanticConfig{
BlacklistTypes: []string{
"AsyncFunctionDef",
"Attribute",
"BoolLiteral",
"Bytes",
"FunctionDef",
Expand Down
197 changes: 144 additions & 53 deletions driver/normalizer/normalizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,43 @@ var Normalize = Transformers([][]Transformer{

func funcDefMap(typ string, async bool) Mapping {
return MapSemantic(typ, uast.FunctionGroup{}, MapObj(
Obj{
"body": Var("body"),
"name": Var("name"),
Fields{
{Name: "body", Op: Var("body")},
{Name: "name", Op: Var("name")},
// Arguments should be converted by the uast.Arguments normalization
"args": Obj{
{Name: "args", Op: Obj{
"args": Var("arguments"),
uast.KeyPos: Var("_pos"),
uast.KeyType: Var("_type"),
}},
// Will be filled only if there is a Python3 type annotation. If the field is
// missing or the value is nil it DOESNT mean that the function returns None,
// just that there is no annotation (but the function could still return some
// other type!).
{Name: "returns", Optional: "ret_opt", Op: Cases("ret_case",
Is(nil),
Obj{
uast.KeyType: String("BoxedName"),
"boxed_value": Var("ret_type"),
// No problem dropping this one, it's used by an internal interpreter optimization/cache
// without semantic meaning
"ctx": Any(),
}),
},
{Name: "decorator_list", Op: Var("func_decorators")},
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
},
Obj{
"Nodes": Arr(
Obj{
Fields{
// FIXME: generator=true if it uses yield anywhere in the body
"async": Bool(async),
{Name: "async", Op: Bool(async)},
{Name: "decorators", Op: Var("func_decorators")},
{Name: "comments", Op: Fields{
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
}},
},
UASTType(uast.Alias{}, Obj{
// FIXME: can't call identifierWithPos because it would take the position of the
Expand All @@ -51,8 +73,17 @@ func funcDefMap(typ string, async bool) Mapping {
"Name": Var("name"),
}),
"Node": UASTType(uast.Function{}, Obj{
"Type": UASTType(uast.FunctionType{}, Obj{
"Arguments": Var("arguments"),
"Type": UASTType(uast.FunctionType{}, Fields{
{Name: "Arguments", Op: Var("arguments")},
{Name: "Returns", Optional: "ret_opt", Op: Cases("ret_case",
// Dont add None here as default, read the comment on the upper
// side of the annotation
Is(nil),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Python functions always have an implicit return, so should be a single empty Argument with Init = None when there are no type annotations. There is a similar piece of code in JS).

Copy link
Contributor Author

@juanjux juanjux Feb 15, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I implemented this and it worked beautifully... but then I noticed that this annotations only has a value IF the function has a Python 3.4+ type annotation BUT if it doesn't have it, it doesn't mean that the function returns None! Just that it's not annotated. So I think is better to left it with the current state (nil if there is no annotation, the annotation if there is one) because otherwise the UAST user could be confused into thinking that some functions return None while in fact they return other types.

Edit: will add a comment to clarify this.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't meant that the function returns None as type, only that if there is no return, all Python2 and Python3 functions returns None implicitly. So it's Returns.Argument.Init: None, same way as we do it in JS with undefined.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but to know if the function returns None implicitly we should parse the function and find the return statement in this case.

Arr(UASTType(uast.Argument{},
Obj{
"Type": Var("ret_type"),
},
)))},
}),
"Body": UASTType(uast.Block{}, Obj{
"Statements": Var("body"),
Expand Down Expand Up @@ -89,7 +120,6 @@ func mapStr(nativeType string) Mapping {
{Name: "boxed_value", Op: UASTType(uast.String{}, Obj{
uast.KeyPos: Var("pos_"),
"Value": Var("s"),
//"Format": String(""),
})},
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
Expand All @@ -99,9 +129,11 @@ func mapStr(nativeType string) Mapping {

var Normalizers = []Mapping{

// Box Names, Strings, and Bools into a "BoxedFoo" moving the real node to the
// Box Names, Strings, Attributes, and Bools into a "BoxedFoo" moving the real node to the
// "value" property and keeping the comments in the parent (if not, comments would be lost
// when promoting the objects)
// when promoting the objects).
// For other objects, the comments are dropped.
// See: https://github.com/bblfsh/sdk/issues/361
Map(
Part("_", Fields{
{Name: uast.KeyType, Op: String("Name")},
Expand All @@ -121,13 +153,10 @@ var Normalizers = []Mapping{
}),
),

// TODO: uncomment after SDK 2.13.x update
// (upgrade currently blocked by: https://github.com/bblfsh/sdk/issues/353)
Map(
Part("_", Fields{
{Name: uast.KeyType, Op: String("BoolLiteral")},
{Name: uast.KeyPos, Op: Var("pos_")},
//{Name: "LiteralValue", Op: Var("lv")},
{Name: "value", Op: Var("lv")},
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
Expand All @@ -143,6 +172,29 @@ var Normalizers = []Mapping{
}),
),

Map(
Part("_", Fields{
{Name: uast.KeyType, Op: String("Attribute")},
{Name: uast.KeyPos, Op: Var("pos_")},
{Name: "attr", Op: Var("aname")},
// No problem dropping this one, it's used by an internal interpreter optimization
// cache without semantic meaning
// without semantic meaning
{Name: "ctx", Op: Any()},
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
}),
Part("_", Fields{
{Name: uast.KeyType, Op: String("BoxedAttribute")},
{Name: "boxed_value", Op: UASTType(uast.Identifier{}, Obj{
uast.KeyPos: Var("pos_"),
"Name": Var("aname"),
})},
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
}),
),

mapStr("Bytes"),
mapStr("Str"),
mapStr("StringLiteral"),
Expand All @@ -165,6 +217,11 @@ var Normalizers = []Mapping{
AnnotateType("keyword", MapObj(
Fields{
{Name: "arg", Op: Var("name")},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
},
Fields{
{Name: "arg",
Expand All @@ -174,35 +231,40 @@ var Normalizers = []Mapping{
}),
role.Name),

MapSemantic("Attribute", uast.Identifier{}, MapObj(
Obj{"attr": Var("name")},
Obj{"Name": Var("name")},
)),

MapSemantic("arg", uast.Argument{}, MapObj(
Obj{
uast.KeyToken: Var("name"),
"default": Var("init"),
},
Obj{
"Name": identifierWithPos("name"),
"Init": Var("init"),
},
)),

MapSemantic("arg", uast.Argument{}, MapObj(
Obj{
uast.KeyToken: Var("name"),
Fields{
{Name: uast.KeyToken, Op: Var("name")},
{Name: "default", Optional: "opt_def", Op: Var("init")},
// No problem dropping this one, it's used by an internal interpreter optimization/cache
// without semantic meaning
{Name: "ctx", Optional: "opt_ctx", Op: Any()},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
// This one is pesky - they're ignored by the runtime, could have typing from
// mypy, or could have anything else, so we can assign to the semantic type
{Name: "annotation", Optional: "ann_opt", Op: Any()},
},
Obj{
"Name": identifierWithPos("name"),
Fields{
{Name: "Name", Op: identifierWithPos("name")},
{Name: "Init", Optional: "opt_def", Op: Var("init")},
},
)),

MapSemantic("kwonly_arg", uast.Argument{}, MapObj(
Obj{
uast.KeyToken: Var("name"),
"default": Var("init"),
Fields{
{Name: uast.KeyToken, Op: Var("name")},
{Name: "default", Op: Var("init")},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
// This one is pesky - they're ignored by the runtime, could have typing from
// mypy, or could have anything else, so we can assign to the semantic type
{Name: "annotation", Op: Any()},
},
Obj{
"Init": Var("init"),
Expand All @@ -211,8 +273,16 @@ var Normalizers = []Mapping{
)),

MapSemantic("vararg", uast.Argument{}, MapObj(
Obj{
uast.KeyToken: Var("name"),
Fields{
{Name: uast.KeyToken, Op: Var("name")},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
// This one is pesky - they're ignored by the runtime, could have typing from
// mypy, or could have anything else, so we can assign to the semantic type
{Name: "annotation", Op: Any()},
},
Obj{
"Name": identifierWithPos("name"),
Expand All @@ -221,12 +291,20 @@ var Normalizers = []Mapping{
)),

MapSemantic("kwarg", uast.Argument{}, MapObj(
Obj{
uast.KeyToken: Var("name"),
Fields{
{Name: uast.KeyToken, Op: Var("name")},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
// This one is pesky - they're ignored by the runtime, could have typing from
// mypy, or could have anything else, so we can assign to the semantic type
{Name: "annotation", Op: Any()},
},
Obj{
"Name": identifierWithPos("name"),
"MapVariadic": Bool(true),
"Name": identifierWithPos("name"),
"MapVariadic": Bool(true),
},
)),

Expand Down Expand Up @@ -264,14 +342,17 @@ var Normalizers = []Mapping{
Objs{
{"Node": Obj{}},
{
"Node": UASTType(uast.Identifier{}, Obj{"Name": Var("alias")}),
"Node": UASTType(uast.Identifier{},
Obj{
"Name": Var("alias"),
}),
}},
))),

// Star imports
MapSemantic("ImportFrom", uast.RuntimeImport{}, MapObj(
Obj{
"names": Arr(
Fields{
{Name: "names", Op: Arr(
Obj{
uast.KeyType: String("uast:Alias"),
uast.KeyPos: Var("pos"),
Expand All @@ -281,9 +362,14 @@ var Normalizers = []Mapping{
},
"Node": Obj{},
},
),
"level": Var("level"),
"module": Var("module"),
)},
{Name: "level", Op: Var("level")},
{Name: "module", Op: Var("module")},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
},
Obj{
"All": Bool(true),
Expand All @@ -300,10 +386,15 @@ var Normalizers = []Mapping{
)),

MapSemantic("ImportFrom", uast.RuntimeImport{}, MapObj(
Obj{
"names": Var("names"),
"module": Var("module"),
"level": Var("level"),
Fields{
{Name: "names", Op: Var("names")},
{Name: "module", Op: Var("module")},
{Name: "level", Op: Var("level")},
// FIXME: change this once we've a way to store other nodes on semantic objects
// See: https://github.com/bblfsh/sdk/issues/361
// See: https://github.com/bblfsh/python-driver/issues/178
{Name: "noops_previous", Optional: "np_opt", Op: Any()},
{Name: "noops_sameline", Optional: "ns_opt", Op: Any()},
},
Obj{
"Names": Var("names"),
Expand Down
Loading