From af7af1cb9a827056f8e98fcecd77310abd8c6d55 Mon Sep 17 00:00:00 2001 From: Levi Blackstone Date: Tue, 21 Jan 2020 12:50:08 -0700 Subject: [PATCH] Move YAML decode logic into provider and improve default Helm namespaces (#952) Reintroduce the reverted changed (#941) from #925 and #934 with a few additional fixes related to the changes in #946. The major changes include the following: - Use a runtime invoke to call a common decodeYaml method in the provider rather than using YAML libraries specific to each language. - Use the namespace parameter of helm.v2.Chart as a default, and set it on known namespace-scoped resources. --- CHANGELOG.md | 7 +- pkg/gen/nodejs-templates/helm/v2/helm.ts | 85 ++------------ pkg/gen/nodejs-templates/yaml.ts.mustache | 40 ++----- pkg/gen/python-templates/helm/v2/helm.py | 10 +- pkg/gen/python-templates/yaml.py.mustache | 9 +- pkg/provider/invoke_decode_yaml.go | 73 ++++++++++++ pkg/provider/provider.go | 41 ++++++- sdk/nodejs/helm/v2/helm.ts | 85 ++------------ sdk/nodejs/tests/helm.ts | 111 ------------------ sdk/nodejs/yaml/yaml.ts | 40 ++----- sdk/python/pulumi_kubernetes/helm/v2/helm.py | 10 +- sdk/python/pulumi_kubernetes/yaml.py | 9 +- sdk/python/setup.py | 1 - tests/examples/examples_test.go | 4 +- tests/examples/helm-local/index.ts | 9 +- tests/examples/helm/index.ts | 9 +- tests/examples/python/get/requirements.txt | 2 +- .../python/guestbook/requirements.txt | 2 +- .../python/helm-local/requirements.txt | 2 +- tests/examples/python/helm/requirements.txt | 2 +- tests/examples/python/python_test.go | 10 +- tests/integration/aliases/aliases_test.go | 26 +--- tests/integration/istio/step1/index.ts | 6 +- tests/integration/istio/step1/istio.ts | 6 +- tests/integration/yaml-url/yaml_url_test.go | 2 +- 25 files changed, 203 insertions(+), 398 deletions(-) create mode 100644 pkg/provider/invoke_decode_yaml.go delete mode 100644 sdk/nodejs/tests/helm.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b18ca7759a..478935ebfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ### Improvements - Improve namespaced Kind check. (https://github.com/pulumi/pulumi-kubernetes/pull/947). +- Add helm template `apiVersions` flag. (https://github.com/pulumi/pulumi-kubernetes/pull/894) +- Move YAML decode logic into provider and improve handling of default namespaces for Helm charts. (https://github.com +/pulumi/pulumi-kubernetes/pull/952). ### Bug fixes @@ -28,10 +31,6 @@ - Fix deprecation warnings and docs. (https://github.com/pulumi/pulumi-kubernetes/pull/929). - Fix projection of array-valued output properties in .NET. (https://github.com/pulumi/pulumi-kubernetes/pull/931) -### Improvements - -- Add helm template `apiVersions` flag. (https://github.com/pulumi/pulumi-kubernetes/pull/894) - ## 1.4.1 (December 17, 2019) ### Bug fixes diff --git a/pkg/gen/nodejs-templates/helm/v2/helm.ts b/pkg/gen/nodejs-templates/helm/v2/helm.ts index 49be628f7b..9d2c71f40c 100644 --- a/pkg/gen/nodejs-templates/helm/v2/helm.ts +++ b/pkg/gen/nodejs-templates/helm/v2/helm.ts @@ -217,7 +217,8 @@ export class Chart extends yaml.CollectionComponentResource { maxBuffer: 512 * 1024 * 1024 // 512 MB }, ).toString(); - return this.parseTemplate(yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps); + return this.parseTemplate( + yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace); } catch (e) { // Shed stack trace, only emit the error. throw new pulumi.RunError(e.toString()); @@ -230,93 +231,25 @@ export class Chart extends yaml.CollectionComponentResource { } parseTemplate( - yamlStream: string, + text: string, transformations: ((o: any, opts: pulumi.CustomResourceOptions) => void)[] | undefined, resourcePrefix: string | undefined, dependsOn: pulumi.Resource[], + defaultNamespace: string | undefined, ): pulumi.Output<{ [key: string]: pulumi.CustomResource }> { - // NOTE: We must manually split the YAML stream because of js-yaml#456. Perusing the code - // and the spec, it looks like a YAML stream is delimited by `^---`, though it is difficult - // to know for sure. - // - // NOTE: We use `{json: true, schema: jsyaml.CORE_SCHEMA}` here so that we conform to Helm's - // YAML parsing semantics. Specifically, `json: true` to ensure that a duplicate key - // overrides its predecessory, rather than throwing an exception, and `schema: - // jsyaml.CORE_SCHEMA` to avoid using additional YAML parsing rules not supported by the - // YAML parser used by Kubernetes. - const objs = yamlStream.split(/^---/m) - .map(yaml => jsyaml.safeLoad(yaml, {json: true, schema: jsyaml.CORE_SCHEMA})) - .filter(a => a != null && "kind" in a) - .sort(helmSort); - return yaml.parse( + const promise = pulumi.runtime.invoke( + "kubernetes:yaml:decode", {text, defaultNamespace}, {async: true}); + return pulumi.output(promise).apply<{[key: string]: pulumi.CustomResource}>(p => yaml.parse( { resourcePrefix: resourcePrefix, - yaml: objs.map(o => jsyaml.safeDump(o)), + objs: p.result, transformations: transformations || [], }, { parent: this, dependsOn: dependsOn } - ); + )); } } -// helmSort is a JavaScript implementation of the Helm Kind sorter[1]. It provides a -// best-effort topology of Kubernetes kinds, which in most cases should ensure that resources -// that must be created first, are. -// -// [1]: https://github.com/helm/helm/blob/094b97ab5d7e2f6eda6d0ab0f2ede9cf578c003c/pkg/tiller/kind_sorter.go -/** @ignore */ export function helmSort(a: { kind: string }, b: { kind: string }): number { - const installOrder = [ - "Namespace", - "ResourceQuota", - "LimitRange", - "PodSecurityPolicy", - "Secret", - "ConfigMap", - "StorageClass", - "PersistentVolume", - "PersistentVolumeClaim", - "ServiceAccount", - "CustomResourceDefinition", - "ClusterRole", - "ClusterRoleBinding", - "Role", - "RoleBinding", - "Service", - "DaemonSet", - "Pod", - "ReplicationController", - "ReplicaSet", - "Deployment", - "StatefulSet", - "Job", - "CronJob", - "Ingress", - "APIService" - ]; - - const ordering: { [key: string]: number } = {}; - installOrder.forEach((_, i) => { - ordering[installOrder[i]] = i; - }); - - const aKind = a["kind"]; - const bKind = b["kind"]; - - if (!(aKind in ordering) && !(bKind in ordering)) { - return aKind.localeCompare(bKind); - } - - if (!(aKind in ordering)) { - return 1; - } - - if (!(bKind in ordering)) { - return -1; - } - - return ordering[aKind] - ordering[bKind]; -} - /** * Additional options to customize the fetching of the Helm chart. */ diff --git a/pkg/gen/nodejs-templates/yaml.ts.mustache b/pkg/gen/nodejs-templates/yaml.ts.mustache index f3a19e4118..7654e8ebfd 100644 --- a/pkg/gen/nodejs-templates/yaml.ts.mustache +++ b/pkg/gen/nodejs-templates/yaml.ts.mustache @@ -87,7 +87,7 @@ import * as outputs from "../types/output"; export interface ConfigOpts { /** JavaScript objects representing Kubernetes resources. */ - objs: any[]; + objs: pulumi.Input[]>; /** * A set of transformations to apply to Kubernetes resource definitions before registering @@ -120,14 +120,9 @@ import * as outputs from "../types/output"; resourcePrefix?: string; } - function yamlLoadAll(text: string): any[] { - // NOTE[pulumi-kubernetes#501]: Use `loadAll` with `JSON_SCHEMA` here instead of - // `safeLoadAll` because the latter is incompatible with `JSON_SCHEMA`. It is - // important to use `JSON_SCHEMA` here because the fields of the Kubernetes core - // API types are all tagged with `json:`, and they don't deal very well with things - // like dates. - const jsyaml = require("js-yaml"); - return jsyaml.loadAll(text, undefined, {schema: jsyaml.JSON_SCHEMA}); + function yamlLoadAll(text: string): Promise { + const promise = pulumi.runtime.invoke("kubernetes:yaml:decode", {text}, {async: true}); + return promise.then(p => p.result); } /** @ignore */ export function parse( @@ -187,9 +182,9 @@ import * as outputs from "../types/output"; } if (config.objs !== undefined) { - const objs= Array.isArray(config.objs) ? config.objs: [config.objs]; - const docResources = parseYamlDocument({objs: objs, transformations: config.transformations}, opts); - resources = pulumi.all([resources, docResources]).apply(([rs, drs]) => ({...rs, ...drs})); + const objs = Array.isArray(config.objs) ? config.objs: [config.objs]; + const docResources = parseYamlDocument({objs, transformations: config.transformations}, opts); + resources = pulumi.all([resources, docResources]).apply(([rs, drs]) => ({...rs, ...drs})); } return resources; @@ -340,22 +335,13 @@ import * as outputs from "../types/output"; config: ConfigOpts, opts?: pulumi.CustomResourceOptions, ): pulumi.Output<{[key: string]: pulumi.CustomResource}> { - const objs: pulumi.Output<{name: string, resource: pulumi.CustomResource}>[] = []; - - for (const obj of config.objs) { - const fileObjects: pulumi.Output<{name: string, resource: pulumi.CustomResource}>[] = - parseYamlObject(obj, config.transformations, config.resourcePrefix, opts); - for (const fileObject of fileObjects) { - objs.push(fileObject); - } - } - return pulumi.all(objs).apply(xs => { - let resources: {[key: string]: pulumi.CustomResource} = {}; - for (const x of xs) { - resources[x.name] = x.resource - } + return pulumi.output(config.objs).apply(configObjs => { + const objs = configObjs + .map(obj => parseYamlObject(obj, config.transformations, config.resourcePrefix, opts)) + .reduce((array, objs) => (array.concat(...objs)), []); - return resources; + return pulumi.output(objs).apply(objs => objs + .reduce((map: {[key: string]: pulumi.CustomResource}, val) => (map[val.name] = val.resource, map), {})) }); } diff --git a/pkg/gen/python-templates/helm/v2/helm.py b/pkg/gen/python-templates/helm/v2/helm.py index 4451a36e5f..63aedf4462 100644 --- a/pkg/gen/python-templates/helm/v2/helm.py +++ b/pkg/gen/python-templates/helm/v2/helm.py @@ -9,7 +9,6 @@ from typing import Any, Callable, List, Optional, TextIO, Tuple, Union import pulumi.runtime -import yaml from pulumi_kubernetes.yaml import _parse_yaml_document @@ -348,10 +347,13 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi cmd.extend(home_arg) chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd) + objects = chart_resources.apply( + lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', { + 'text': text, 'defaultNamespace': config.namespace}).value['result']) # Parse the manifest and create the specified resources. - resources = chart_resources.apply( - lambda yaml_str: _parse_yaml_document(yaml.safe_load_all(yaml_str), opts, config.transformations)) + resources = objects.apply( + lambda objects: _parse_yaml_document(objects, opts, config.transformations)) pulumi.Output.all(file, chart_dir, resources).apply(_cleanup_temp_dir) return resources @@ -472,7 +474,7 @@ def get_resource(self, group_version_kind, name, namespace=None) -> pulumi.Outpu # `id` will either be `${name}` or `${namespace}/${name}`. id = pulumi.Output.from_input(name) - if namespace != None: + if namespace is not None: id = pulumi.Output.concat(namespace, '/', name) resource_id = id.apply(lambda x: f'{group_version_kind}:{x}') diff --git a/pkg/gen/python-templates/yaml.py.mustache b/pkg/gen/python-templates/yaml.py.mustache index ed62d9ebcc..1294b03d4e 100644 --- a/pkg/gen/python-templates/yaml.py.mustache +++ b/pkg/gen/python-templates/yaml.py.mustache @@ -8,8 +8,6 @@ from typing import Callable, Dict, List, Optional import pulumi.runtime import requests -import yaml -import pulumi_kubernetes from pulumi_kubernetes.apiextensions import CustomResource from . import tables @@ -25,7 +23,6 @@ class ConfigFile(pulumi.ComponentResource): Kubernetes resources contained in this ConfigFile. """ - def __init__(self, name, file_id, opts=None, transformations=None, resource_prefix=None): """ :param str name: A name for a resource. @@ -63,7 +60,8 @@ class ConfigFile(pulumi.ComponentResource): # Note: Unlike NodeJS, Python requires that we "pull" on our futures in order to get them scheduled for # execution. In order to do this, we leverage the engine's RegisterResourceOutputs to wait for the # resolution of all resources that this YAML document created. - self.resources = _parse_yaml_document(yaml.safe_load_all(text), opts, transformations, resource_prefix) + __ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'] + self.resources = _parse_yaml_document(__ret__, opts, transformations, resource_prefix) self.register_outputs({"resources": self.resources}) def translate_output_property(self, prop: str) -> str: @@ -84,12 +82,13 @@ class ConfigFile(pulumi.ComponentResource): # `id` will either be `${name}` or `${namespace}/${name}`. id = pulumi.Output.from_input(name) - if namespace != None: + if namespace is not None: id = pulumi.Output.concat(namespace, '/', name) resource_id = id.apply(lambda x: f'{group_version_kind}:{x}') return resource_id.apply(lambda x: self.resources[x]) + def _read_url(url: str) -> str: response = requests.get(url) response.raise_for_status() diff --git a/pkg/provider/invoke_decode_yaml.go b/pkg/provider/invoke_decode_yaml.go new file mode 100644 index 0000000000..bfde8296e7 --- /dev/null +++ b/pkg/provider/invoke_decode_yaml.go @@ -0,0 +1,73 @@ +// Copyright 2016-2019, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "io" + "io/ioutil" + "strings" + + "github.com/pulumi/pulumi-kubernetes/pkg/clients" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// decodeYaml parses a YAML string, and then returns a slice of untyped structs that can be marshalled into +// Pulumi RPC calls. If a default namespace is specified, set that on the relevant decoded objects. +func decodeYaml(text, defaultNamespace string, clientSet *clients.DynamicClientSet) ([]interface{}, error) { + var resources []unstructured.Unstructured + + dec := yaml.NewYAMLOrJSONDecoder(ioutil.NopCloser(strings.NewReader(text)), 128) + for { + var value map[string]interface{} + if err := dec.Decode(&value); err != nil { + if err == io.EOF { + break + } + return nil, err + } + resource := unstructured.Unstructured{Object: value} + + // Sometimes manifests include empty resources, so skip these. + if len(resource.GetKind()) == 0 || len(resource.GetAPIVersion()) == 0 { + continue + } + + if len(defaultNamespace) > 0 { + namespaced, err := clients.IsNamespacedKind(resource.GroupVersionKind(), clientSet) + if err != nil { + if clients.IsNoNamespaceInfoErr(err) { + // Assume resource is namespaced. + namespaced = true + } else { + return nil, err + } + } + + // Set namespace if resource Kind is namespaced and namespace is not already set. + if namespaced && len(resource.GetNamespace()) == 0 { + resource.SetNamespace(defaultNamespace) + } + } + resources = append(resources, resource) + } + + result := make([]interface{}, len(resources)) + for _, resource := range resources { + result = append(result, resource.Object) + } + + return result, nil +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index c163b08494..512e2acca4 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -72,6 +72,7 @@ const ( streamInvokeList = "kubernetes:kubernetes:list" streamInvokeWatch = "kubernetes:kubernetes:watch" streamInvokePodLogs = "kubernetes:kubernetes:podLogs" + invokeDecodeYaml = "kubernetes:yaml:decode" lastAppliedConfigKey = "kubectl.kubernetes.io/last-applied-configuration" initialApiVersionKey = "__initialApiVersion" ) @@ -389,9 +390,45 @@ func (k *kubeProvider) Invoke(ctx context.Context, // inputs may not have resolved yet. Any invoke logic that depends on an active cluster must check // k.clusterUnreachable and handle that condition appropriately. - // Always fail. tok := req.GetTok() - return nil, fmt.Errorf("unknown Invoke type '%s'", tok) + label := fmt.Sprintf("%s.Invoke(%s)", k.label(), tok) + args, err := plugin.UnmarshalProperties( + req.GetArgs(), plugin.MarshalOptions{Label: label, KeepUnknowns: true}) + if err != nil { + return nil, pkgerrors.Wrapf(err, "failed to unmarshal %v args during an Invoke call", tok) + } + + switch tok { + case invokeDecodeYaml: + var text, defaultNamespace string + if textArg := args["text"]; textArg.HasValue() && textArg.IsString() { + text = textArg.StringValue() + } else { + return nil, pkgerrors.New("missing required field 'text' of type string") + } + if defaultNsArg := args["defaultNamespace"]; defaultNsArg.HasValue() && defaultNsArg.IsString() { + defaultNamespace = defaultNsArg.StringValue() + } + + result, err := decodeYaml(text, defaultNamespace, k.clientSet) + if err != nil { + return nil, err + } + + objProps, err := plugin.MarshalProperties( + resource.NewPropertyMapFromMap(map[string]interface{}{"result": result}), + plugin.MarshalOptions{ + Label: label, KeepUnknowns: true, SkipNulls: true, + }) + if err != nil { + return nil, err + } + + return &pulumirpc.InvokeResponse{Return: objProps}, nil + + default: + return nil, fmt.Errorf("unknown Invoke type %q", tok) + } } // StreamInvoke dynamically executes a built-in function in the provider. The result is streamed diff --git a/sdk/nodejs/helm/v2/helm.ts b/sdk/nodejs/helm/v2/helm.ts index 49be628f7b..9d2c71f40c 100644 --- a/sdk/nodejs/helm/v2/helm.ts +++ b/sdk/nodejs/helm/v2/helm.ts @@ -217,7 +217,8 @@ export class Chart extends yaml.CollectionComponentResource { maxBuffer: 512 * 1024 * 1024 // 512 MB }, ).toString(); - return this.parseTemplate(yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps); + return this.parseTemplate( + yamlStream, cfg.transformations, cfg.resourcePrefix, configDeps, cfg.namespace); } catch (e) { // Shed stack trace, only emit the error. throw new pulumi.RunError(e.toString()); @@ -230,93 +231,25 @@ export class Chart extends yaml.CollectionComponentResource { } parseTemplate( - yamlStream: string, + text: string, transformations: ((o: any, opts: pulumi.CustomResourceOptions) => void)[] | undefined, resourcePrefix: string | undefined, dependsOn: pulumi.Resource[], + defaultNamespace: string | undefined, ): pulumi.Output<{ [key: string]: pulumi.CustomResource }> { - // NOTE: We must manually split the YAML stream because of js-yaml#456. Perusing the code - // and the spec, it looks like a YAML stream is delimited by `^---`, though it is difficult - // to know for sure. - // - // NOTE: We use `{json: true, schema: jsyaml.CORE_SCHEMA}` here so that we conform to Helm's - // YAML parsing semantics. Specifically, `json: true` to ensure that a duplicate key - // overrides its predecessory, rather than throwing an exception, and `schema: - // jsyaml.CORE_SCHEMA` to avoid using additional YAML parsing rules not supported by the - // YAML parser used by Kubernetes. - const objs = yamlStream.split(/^---/m) - .map(yaml => jsyaml.safeLoad(yaml, {json: true, schema: jsyaml.CORE_SCHEMA})) - .filter(a => a != null && "kind" in a) - .sort(helmSort); - return yaml.parse( + const promise = pulumi.runtime.invoke( + "kubernetes:yaml:decode", {text, defaultNamespace}, {async: true}); + return pulumi.output(promise).apply<{[key: string]: pulumi.CustomResource}>(p => yaml.parse( { resourcePrefix: resourcePrefix, - yaml: objs.map(o => jsyaml.safeDump(o)), + objs: p.result, transformations: transformations || [], }, { parent: this, dependsOn: dependsOn } - ); + )); } } -// helmSort is a JavaScript implementation of the Helm Kind sorter[1]. It provides a -// best-effort topology of Kubernetes kinds, which in most cases should ensure that resources -// that must be created first, are. -// -// [1]: https://github.com/helm/helm/blob/094b97ab5d7e2f6eda6d0ab0f2ede9cf578c003c/pkg/tiller/kind_sorter.go -/** @ignore */ export function helmSort(a: { kind: string }, b: { kind: string }): number { - const installOrder = [ - "Namespace", - "ResourceQuota", - "LimitRange", - "PodSecurityPolicy", - "Secret", - "ConfigMap", - "StorageClass", - "PersistentVolume", - "PersistentVolumeClaim", - "ServiceAccount", - "CustomResourceDefinition", - "ClusterRole", - "ClusterRoleBinding", - "Role", - "RoleBinding", - "Service", - "DaemonSet", - "Pod", - "ReplicationController", - "ReplicaSet", - "Deployment", - "StatefulSet", - "Job", - "CronJob", - "Ingress", - "APIService" - ]; - - const ordering: { [key: string]: number } = {}; - installOrder.forEach((_, i) => { - ordering[installOrder[i]] = i; - }); - - const aKind = a["kind"]; - const bKind = b["kind"]; - - if (!(aKind in ordering) && !(bKind in ordering)) { - return aKind.localeCompare(bKind); - } - - if (!(aKind in ordering)) { - return 1; - } - - if (!(bKind in ordering)) { - return -1; - } - - return ordering[aKind] - ordering[bKind]; -} - /** * Additional options to customize the fetching of the Helm chart. */ diff --git a/sdk/nodejs/tests/helm.ts b/sdk/nodejs/tests/helm.ts deleted file mode 100644 index fdc917496e..0000000000 --- a/sdk/nodejs/tests/helm.ts +++ /dev/null @@ -1,111 +0,0 @@ -import * as assert from "assert"; -import * as helm from "../helm/index"; -const helmSort = helm.v2.helmSort; - -function makeKinds(kinds: string[]): { kind: string }[] { - return kinds.map(kind => { - return { kind: kind }; - }); -} - -// Simple Fischer-Yates implementation. Taken from StackOverflow[1]. -// -// [1]: https://stackoverflow.com/a/6274381 -function shuffle(a: T[]): T[] { - for (let i = a.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [a[i], a[j]] = [a[j], a[i]]; - } - return a; -} - -describe("helmSort", () => { - it("is the identity function for the empty array", () => { - assert.deepStrictEqual([].sort(helmSort), []); - }); - - it("is the identity function for single-element array", () => { - assert.deepStrictEqual(makeKinds(["Namespace"]).sort(helmSort), makeKinds(["Namespace"])); - }); - - it("correctly sorts duplicates", () => { - assert.deepStrictEqual( - makeKinds(["Namespace", "LimitRange", "Namespace"]).sort(helmSort), - makeKinds(["Namespace", "Namespace", "LimitRange"]) - ); - }); - - it("puts unknown kinds last", () => { - assert.deepStrictEqual( - makeKinds(["UNKNOWN KIND", "LimitRange", "Namespace"]).sort(helmSort), - makeKinds(["Namespace", "LimitRange", "UNKNOWN KIND"]) - ); - }); - - it("sorts a shuffled array", () => { - const shuffled = makeKinds( - shuffle([ - "Namespace", - "ResourceQuota", - "LimitRange", - "PodSecurityPolicy", - "Secret", - "ConfigMap", - "StorageClass", - "PersistentVolume", - "PersistentVolumeClaim", - "ServiceAccount", - "CustomResourceDefinition", - "ClusterRole", - "ClusterRoleBinding", - "Role", - "RoleBinding", - "Service", - "DaemonSet", - "Pod", - "ReplicationController", - "ReplicaSet", - "Deployment", - "StatefulSet", - "Job", - "CronJob", - "Ingress", - "APIService", - "UNKNOWN_KIND" - ]) - ); - - const sorted = makeKinds([ - "Namespace", - "ResourceQuota", - "LimitRange", - "PodSecurityPolicy", - "Secret", - "ConfigMap", - "StorageClass", - "PersistentVolume", - "PersistentVolumeClaim", - "ServiceAccount", - "CustomResourceDefinition", - "ClusterRole", - "ClusterRoleBinding", - "Role", - "RoleBinding", - "Service", - "DaemonSet", - "Pod", - "ReplicationController", - "ReplicaSet", - "Deployment", - "StatefulSet", - "Job", - "CronJob", - "Ingress", - "APIService", - "UNKNOWN_KIND" - ]); - - assert.notDeepStrictEqual(shuffled, sorted); - assert.deepStrictEqual(shuffled.sort(helmSort), sorted); - }); -}); diff --git a/sdk/nodejs/yaml/yaml.ts b/sdk/nodejs/yaml/yaml.ts index fa4841a79c..d99b44525f 100644 --- a/sdk/nodejs/yaml/yaml.ts +++ b/sdk/nodejs/yaml/yaml.ts @@ -87,7 +87,7 @@ import * as outputs from "../types/output"; export interface ConfigOpts { /** JavaScript objects representing Kubernetes resources. */ - objs: any[]; + objs: pulumi.Input[]>; /** * A set of transformations to apply to Kubernetes resource definitions before registering @@ -120,14 +120,9 @@ import * as outputs from "../types/output"; resourcePrefix?: string; } - function yamlLoadAll(text: string): any[] { - // NOTE[pulumi-kubernetes#501]: Use `loadAll` with `JSON_SCHEMA` here instead of - // `safeLoadAll` because the latter is incompatible with `JSON_SCHEMA`. It is - // important to use `JSON_SCHEMA` here because the fields of the Kubernetes core - // API types are all tagged with `json:`, and they don't deal very well with things - // like dates. - const jsyaml = require("js-yaml"); - return jsyaml.loadAll(text, undefined, {schema: jsyaml.JSON_SCHEMA}); + function yamlLoadAll(text: string): Promise { + const promise = pulumi.runtime.invoke("kubernetes:yaml:decode", {text}, {async: true}); + return promise.then(p => p.result); } /** @ignore */ export function parse( @@ -187,9 +182,9 @@ import * as outputs from "../types/output"; } if (config.objs !== undefined) { - const objs= Array.isArray(config.objs) ? config.objs: [config.objs]; - const docResources = parseYamlDocument({objs: objs, transformations: config.transformations}, opts); - resources = pulumi.all([resources, docResources]).apply(([rs, drs]) => ({...rs, ...drs})); + const objs = Array.isArray(config.objs) ? config.objs: [config.objs]; + const docResources = parseYamlDocument({objs, transformations: config.transformations}, opts); + resources = pulumi.all([resources, docResources]).apply(([rs, drs]) => ({...rs, ...drs})); } return resources; @@ -2482,22 +2477,13 @@ import * as outputs from "../types/output"; config: ConfigOpts, opts?: pulumi.CustomResourceOptions, ): pulumi.Output<{[key: string]: pulumi.CustomResource}> { - const objs: pulumi.Output<{name: string, resource: pulumi.CustomResource}>[] = []; - - for (const obj of config.objs) { - const fileObjects: pulumi.Output<{name: string, resource: pulumi.CustomResource}>[] = - parseYamlObject(obj, config.transformations, config.resourcePrefix, opts); - for (const fileObject of fileObjects) { - objs.push(fileObject); - } - } - return pulumi.all(objs).apply(xs => { - let resources: {[key: string]: pulumi.CustomResource} = {}; - for (const x of xs) { - resources[x.name] = x.resource - } + return pulumi.output(config.objs).apply(configObjs => { + const objs = configObjs + .map(obj => parseYamlObject(obj, config.transformations, config.resourcePrefix, opts)) + .reduce((array, objs) => (array.concat(...objs)), []); - return resources; + return pulumi.output(objs).apply(objs => objs + .reduce((map: {[key: string]: pulumi.CustomResource}, val) => (map[val.name] = val.resource, map), {})) }); } diff --git a/sdk/python/pulumi_kubernetes/helm/v2/helm.py b/sdk/python/pulumi_kubernetes/helm/v2/helm.py index 4451a36e5f..63aedf4462 100644 --- a/sdk/python/pulumi_kubernetes/helm/v2/helm.py +++ b/sdk/python/pulumi_kubernetes/helm/v2/helm.py @@ -9,7 +9,6 @@ from typing import Any, Callable, List, Optional, TextIO, Tuple, Union import pulumi.runtime -import yaml from pulumi_kubernetes.yaml import _parse_yaml_document @@ -348,10 +347,13 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi cmd.extend(home_arg) chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd) + objects = chart_resources.apply( + lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', { + 'text': text, 'defaultNamespace': config.namespace}).value['result']) # Parse the manifest and create the specified resources. - resources = chart_resources.apply( - lambda yaml_str: _parse_yaml_document(yaml.safe_load_all(yaml_str), opts, config.transformations)) + resources = objects.apply( + lambda objects: _parse_yaml_document(objects, opts, config.transformations)) pulumi.Output.all(file, chart_dir, resources).apply(_cleanup_temp_dir) return resources @@ -472,7 +474,7 @@ def get_resource(self, group_version_kind, name, namespace=None) -> pulumi.Outpu # `id` will either be `${name}` or `${namespace}/${name}`. id = pulumi.Output.from_input(name) - if namespace != None: + if namespace is not None: id = pulumi.Output.concat(namespace, '/', name) resource_id = id.apply(lambda x: f'{group_version_kind}:{x}') diff --git a/sdk/python/pulumi_kubernetes/yaml.py b/sdk/python/pulumi_kubernetes/yaml.py index 93b25b1c3a..af4934af11 100644 --- a/sdk/python/pulumi_kubernetes/yaml.py +++ b/sdk/python/pulumi_kubernetes/yaml.py @@ -8,8 +8,6 @@ import pulumi.runtime import requests -import yaml -import pulumi_kubernetes from pulumi_kubernetes.apiextensions import CustomResource from . import tables @@ -25,7 +23,6 @@ class ConfigFile(pulumi.ComponentResource): Kubernetes resources contained in this ConfigFile. """ - def __init__(self, name, file_id, opts=None, transformations=None, resource_prefix=None): """ :param str name: A name for a resource. @@ -63,7 +60,8 @@ def __init__(self, name, file_id, opts=None, transformations=None, resource_pref # Note: Unlike NodeJS, Python requires that we "pull" on our futures in order to get them scheduled for # execution. In order to do this, we leverage the engine's RegisterResourceOutputs to wait for the # resolution of all resources that this YAML document created. - self.resources = _parse_yaml_document(yaml.safe_load_all(text), opts, transformations, resource_prefix) + __ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'] + self.resources = _parse_yaml_document(__ret__, opts, transformations, resource_prefix) self.register_outputs({"resources": self.resources}) def translate_output_property(self, prop: str) -> str: @@ -84,12 +82,13 @@ def get_resource(self, group_version_kind, name, namespace=None) -> pulumi.Outpu # `id` will either be `${name}` or `${namespace}/${name}`. id = pulumi.Output.from_input(name) - if namespace != None: + if namespace is not None: id = pulumi.Output.concat(namespace, '/', name) resource_id = id.apply(lambda x: f'{group_version_kind}:{x}') return resource_id.apply(lambda x: self.resources[x]) + def _read_url(url: str) -> str: response = requests.get(url) response.raise_for_status() diff --git a/sdk/python/setup.py b/sdk/python/setup.py index e7a20cc354..6a68897e85 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -33,7 +33,6 @@ def readme(): install_requires=[ 'pulumi>=1.4.0,<2.0.0', 'requests>=2.21.0,<2.22.0', - 'pyyaml>=5.1,<5.2', 'semver>=2.8.1', 'parver>=0.2.1', ], diff --git a/tests/examples/examples_test.go b/tests/examples/examples_test.go index cb18999803..a34278f4a4 100644 --- a/tests/examples/examples_test.go +++ b/tests/examples/examples_test.go @@ -164,7 +164,7 @@ func TestAccHelmApiVersions(t *testing.T) { t *testing.T, stackInfo integration.RuntimeValidationStackInfo, ) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 6, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 7, len(stackInfo.Deployment.Resources)) }, }) @@ -181,7 +181,7 @@ func TestAccHelmLocal(t *testing.T) { t *testing.T, stackInfo integration.RuntimeValidationStackInfo, ) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 15, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 16, len(stackInfo.Deployment.Resources)) }, }) diff --git a/tests/examples/helm-local/index.ts b/tests/examples/helm-local/index.ts index 7f1a9f44bf..cd900b053d 100644 --- a/tests/examples/helm-local/index.ts +++ b/tests/examples/helm-local/index.ts @@ -23,6 +23,7 @@ function chart(resourcePrefix?: string): k8s.helm.v2.Chart { // Represents chart `stable/nginx-lego@v0.3.1`. path: "nginx-lego", version: "0.3.1", + namespace: namespaceName, resourcePrefix: resourcePrefix, values: { // Override for the Chart's `values.yml` file. Use `null` to zero out resource requests so it @@ -42,14 +43,6 @@ function chart(resourcePrefix?: string): k8s.helm.v2.Chart { } } }, - // Put every resource in the created namespace. - (obj: any) => { - if (obj.metadata !== undefined) { - obj.metadata.namespace = namespaceName; - } else { - obj.metadata = {namespace: namespaceName}; - } - } ] }); } diff --git a/tests/examples/helm/index.ts b/tests/examples/helm/index.ts index 032621bbb3..aa068330ee 100644 --- a/tests/examples/helm/index.ts +++ b/tests/examples/helm/index.ts @@ -23,6 +23,7 @@ const nginx = new k8s.helm.v2.Chart("simple-nginx", { repo: "stable", chart: "nginx-lego", version: "0.3.1", + namespace: namespaceName, values: { // Override for the Chart's `values.yml` file. Use `null` to zero out resource requests so it // can be scheduled on our (wimpy) CI cluster. (Setting these values to `null` is the "normal" @@ -41,14 +42,6 @@ const nginx = new k8s.helm.v2.Chart("simple-nginx", { } } }, - // Put every resource in the created namespace. - (obj: any) => { - if (obj.metadata !== undefined) { - obj.metadata.namespace = namespaceName - } else { - obj.metadata = {namespace: namespaceName} - } - }, (obj: any, opts: pulumi.CustomResourceOptions) => { if (obj.kind == "Service" && obj.apiVersion == "v1") { opts.additionalSecretOutputs = ["status"]; diff --git a/tests/examples/python/get/requirements.txt b/tests/examples/python/get/requirements.txt index 05dd029e0d..2ee35c6fc7 100644 --- a/tests/examples/python/get/requirements.txt +++ b/tests/examples/python/get/requirements.txt @@ -1 +1 @@ -pulumi>=1.0.0b3,<1.0.1 +pulumi>=1.8.1,<2.0.0 diff --git a/tests/examples/python/guestbook/requirements.txt b/tests/examples/python/guestbook/requirements.txt index 728f25ef0b..2ee35c6fc7 100644 --- a/tests/examples/python/guestbook/requirements.txt +++ b/tests/examples/python/guestbook/requirements.txt @@ -1 +1 @@ -pulumi>=1.0.0b3,<1.0.1 \ No newline at end of file +pulumi>=1.8.1,<2.0.0 diff --git a/tests/examples/python/helm-local/requirements.txt b/tests/examples/python/helm-local/requirements.txt index 728f25ef0b..2ee35c6fc7 100644 --- a/tests/examples/python/helm-local/requirements.txt +++ b/tests/examples/python/helm-local/requirements.txt @@ -1 +1 @@ -pulumi>=1.0.0b3,<1.0.1 \ No newline at end of file +pulumi>=1.8.1,<2.0.0 diff --git a/tests/examples/python/helm/requirements.txt b/tests/examples/python/helm/requirements.txt index 2dd026c0fa..444f651266 100644 --- a/tests/examples/python/helm/requirements.txt +++ b/tests/examples/python/helm/requirements.txt @@ -1,2 +1,2 @@ -pulumi>=1.0.0b3,<1.0.1 +pulumi>=1.8.1,<2.0.0 pulumi_random>==1.0.1a1566600005 diff --git a/tests/examples/python/python_test.go b/tests/examples/python/python_test.go index 5bfa522ca9..fc0f3fc763 100644 --- a/tests/examples/python/python_test.go +++ b/tests/examples/python/python_test.go @@ -85,7 +85,7 @@ func TestYaml(t *testing.T) { ExpectRefreshChanges: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 14, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 15, len(stackInfo.Deployment.Resources)) sort.Slice(stackInfo.Deployment.Resources, func(i, j int) bool { ri := stackInfo.Deployment.Resources[i] @@ -156,12 +156,14 @@ func TestYaml(t *testing.T) { // Note: Skipping validation for the guestbook app in this test since it's no different from the // first ConfigFile. - // Verify the provider resource. + // Verify the provider resources. provRes := stackInfo.Deployment.Resources[12] assert.True(t, providers.IsProviderType(provRes.URN.Type())) + provRes = stackInfo.Deployment.Resources[13] + assert.True(t, providers.IsProviderType(provRes.URN.Type())) // Verify root resource. - stackRes := stackInfo.Deployment.Resources[13] + stackRes := stackInfo.Deployment.Resources[14] assert.Equal(t, resource.RootStackType, stackRes.URN.Type()) // TODO[pulumi/pulumi#2782] Testing of secrets blocked on a bug in Python support for secrets. @@ -310,7 +312,7 @@ func TestHelm(t *testing.T) { ExpectRefreshChanges: true, // PodDisruptionBudget status gets updated by the Deployment. ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 15, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 16, len(stackInfo.Deployment.Resources)) sort.Slice(stackInfo.Deployment.Resources, func(i, j int) bool { ri := stackInfo.Deployment.Resources[i] diff --git a/tests/integration/aliases/aliases_test.go b/tests/integration/aliases/aliases_test.go index 5acc54af04..06f3f79610 100644 --- a/tests/integration/aliases/aliases_test.go +++ b/tests/integration/aliases/aliases_test.go @@ -20,8 +20,6 @@ import ( "github.com/pulumi/pulumi-kubernetes/pkg/openapi" "github.com/pulumi/pulumi-kubernetes/tests" - "github.com/pulumi/pulumi/pkg/resource" - "github.com/pulumi/pulumi/pkg/resource/deploy/providers" "github.com/pulumi/pulumi/pkg/testing/integration" "github.com/stretchr/testify/assert" ) @@ -39,16 +37,10 @@ func TestAliases(t *testing.T) { Quick: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 4, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 5, len(stackInfo.Deployment.Resources)) tests.SortResourcesByURN(stackInfo) - stackRes := stackInfo.Deployment.Resources[3] - assert.Equal(t, resource.RootStackType, stackRes.URN.Type()) - - provRes := stackInfo.Deployment.Resources[2] - assert.True(t, providers.IsProviderType(provRes.URN.Type())) - deployment := stackInfo.Deployment.Resources[0] assert.Equal(t, "alias-test", string(deployment.URN.Name())) assert.Equal(t, "kubernetes:extensions/v1beta1:Deployment", string(deployment.Type)) @@ -59,16 +51,10 @@ func TestAliases(t *testing.T) { Additive: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 4, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 5, len(stackInfo.Deployment.Resources)) tests.SortResourcesByURN(stackInfo) - stackRes := stackInfo.Deployment.Resources[3] - assert.Equal(t, resource.RootStackType, stackRes.URN.Type()) - - provRes := stackInfo.Deployment.Resources[2] - assert.True(t, providers.IsProviderType(provRes.URN.Type())) - deployment := stackInfo.Deployment.Resources[0] assert.Equal(t, "alias-test", string(deployment.URN.Name())) assert.Equal(t, "kubernetes:apps/v1:Deployment", string(deployment.Type)) @@ -83,16 +69,10 @@ func TestAliases(t *testing.T) { Additive: true, ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 4, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 5, len(stackInfo.Deployment.Resources)) tests.SortResourcesByURN(stackInfo) - stackRes := stackInfo.Deployment.Resources[3] - assert.Equal(t, resource.RootStackType, stackRes.URN.Type()) - - provRes := stackInfo.Deployment.Resources[2] - assert.True(t, providers.IsProviderType(provRes.URN.Type())) - deployment := stackInfo.Deployment.Resources[0] assert.Equal(t, "alias-test", string(deployment.URN.Name())) assert.Equal(t, "kubernetes:apps/v1:Deployment", string(deployment.Type)) diff --git a/tests/integration/istio/step1/index.ts b/tests/integration/istio/step1/index.ts index db2b4f6c6a..266be81476 100644 --- a/tests/integration/istio/step1/index.ts +++ b/tests/integration/istio/step1/index.ts @@ -16,7 +16,7 @@ import * as k8s from "@pulumi/kubernetes"; import * as pulumi from "@pulumi/pulumi"; import { k8sProvider } from "./cluster"; -import { istio } from "./istio"; +import { crd10, crd11, crd12, istio } from "./istio"; new k8s.core.v1.Namespace( "bookinfo", @@ -33,13 +33,13 @@ function addNamespace(o: any) { const bookinfo = new k8s.yaml.ConfigFile( "yaml/bookinfo.yaml", { transformations: [addNamespace] }, - { dependsOn: istio, providers: { kubernetes: k8sProvider } } + { dependsOn: [crd10, crd11, crd12], providers: { kubernetes: k8sProvider } } ); new k8s.yaml.ConfigFile( "yaml/bookinfo-gateway.yaml", { transformations: [addNamespace] }, - { dependsOn: bookinfo, providers: { kubernetes: k8sProvider } } + { dependsOn: [crd10, crd11, crd12], providers: { kubernetes: k8sProvider } } ); export const port = istio diff --git a/tests/integration/istio/step1/istio.ts b/tests/integration/istio/step1/istio.ts index 9329acc35d..7d30be37b8 100644 --- a/tests/integration/istio/step1/istio.ts +++ b/tests/integration/istio/step1/istio.ts @@ -52,9 +52,9 @@ export const istio_init = new k8s.helm.v2.Chart( { dependsOn: [namespace, adminBinding], providers: { kubernetes: k8sProvider } } ); -const crd10 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-10"); -const crd11 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-11"); -const crd12 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-12"); +export const crd10 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-10"); +export const crd11 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-11"); +export const crd12 = istio_init.getResource("batch/v1/Job", "istio-system", "istio-init-crd-12"); export const istio = new k8s.helm.v2.Chart( appName, diff --git a/tests/integration/yaml-url/yaml_url_test.go b/tests/integration/yaml-url/yaml_url_test.go index aa9087f196..058357b9b9 100644 --- a/tests/integration/yaml-url/yaml_url_test.go +++ b/tests/integration/yaml-url/yaml_url_test.go @@ -37,7 +37,7 @@ func TestYAMLURL(t *testing.T) { assert.NotNil(t, stackInfo.Deployment) // Assert that we've retrieved the YAML from the URL and provisioned them. - assert.Equal(t, 18, len(stackInfo.Deployment.Resources)) + assert.Equal(t, 19, len(stackInfo.Deployment.Resources)) }, }) }