add destination metadata plugin (#48)

* add destination metadata plugin

* optimizations

* add ManuallyEnableDestination

* add unit tests

* clean up

* add to do item

* bug fix

Author: wenxi-zeng

Analytics-CSharp/Segment/Analytics/Plugins.cs

@@ -92,7 +92,7 @@ public abstract class DestinationPlugin : EventPlugin
         public override PluginType Type => PluginType.Destination;
         public abstract string Key { get; }
 
-        private bool _enabled = false;
+        internal bool _enabled = false;
 
         private readonly Timeline _timeline = new Timeline();
 
@@ -131,7 +131,7 @@ public void Apply(Action<Plugin> closure)
         /// <param name="type">value of <see cref="UpdateType"/>. is the update an initialization or refreshment</param>
         public override void Update(Settings settings, UpdateType type)
         {
-            _enabled = settings.Integrations?.ContainsKey(Key) ?? false;
+            _enabled = Key != null && (settings.Integrations?.ContainsKey(Key) ?? false);
             _timeline.Apply(plugin => plugin.Update(settings, type));
         }
 
@@ -246,5 +246,21 @@ public Plugin Add(Plugin plugin)
         /// <param name="destinationKey">the key of <see cref="DestinationPlugin"/></param>
         /// <returns></returns>
         public DestinationPlugin Find(string destinationKey) => Timeline.Find(destinationKey);
+
+        /// <summary>
+        /// Manually enable a destination plugin.  This is useful when a given DestinationPlugin doesn't have any Segment tie-ins at all.
+        /// This will allow the destination to be processed in the same way within this library.
+        /// </summary>
+        /// <param name="plugin">Destination plugin that needs to be enabled</param>
+        public void ManuallyEnableDestination(DestinationPlugin plugin)
+        {
+            AnalyticsScope.Launch(AnalyticsDispatcher, async () =>
+            {
+                await Store.Dispatch<System.AddDestinationToSettingsAction, System>(
+                    new System.AddDestinationToSettingsAction(plugin.Key));
+            });
+
+            Find(plugin.Key)._enabled = true;
+        }
     }
 }

Analytics-CSharp/Segment/Analytics/Plugins/DestinationMetadataPlugin.cs

@@ -0,0 +1,76 @@
+using System.Collections.Generic;
+using System.Linq;
+using Segment.Serialization;
+
+namespace Segment.Analytics.Plugins
+{
+    public class DestinationMetadataPlugin: Plugin
+    {
+        public override PluginType Type => PluginType.Enrichment;
+
+        private Settings _settings;
+
+        public override void Update(Settings settings, UpdateType type)
+        {
+            base.Update(settings, type);
+            _settings = settings;
+        }
+
+        public override RawEvent Execute(RawEvent incomingEvent)
+        {
+            // TODO: Precompute the metadata instead of compute it for every events once we have Sovran changed to synchronize mode
+            HashSet<string> bundled = new HashSet<string>();
+            HashSet<string> unbundled = new HashSet<string>();
+
+            foreach (Plugin plugin in Analytics.Timeline._plugins[PluginType.Destination]._plugins)
+            {
+                if (plugin is DestinationPlugin destinationPlugin && !(plugin is SegmentDestination) && destinationPlugin._enabled)
+                {
+                    bundled.Add(destinationPlugin.Key);
+                }
+            }
+
+            // All active integrations, not in `bundled` are put in `unbundled`
+            foreach (string integration in _settings.Integrations.Keys)
+            {
+                if (integration != "Segment.io" && !bundled.Contains(integration))
+                {
+                    unbundled.Add(integration);
+                }
+            }
+
+            // All unbundledIntegrations not in `bundled` are put in `unbundled`
+            JsonArray unbundledIntegrations =
+                _settings.Integrations?.GetJsonObject("Segment.io")?.GetJsonArray("unbundledIntegrations") ??
+                new JsonArray();
+            foreach (JsonElement integration in unbundledIntegrations)
+            {
+                string content = ((JsonPrimitive)integration).Content;
+                if (!bundled.Contains(content))
+                {
+                    unbundled.Add(content);
+                }
+            }
+
+            incomingEvent._metadata = new DestinationMetadata
+            {
+                Bundled = CreateJsonArray(bundled),
+                Unbundled = CreateJsonArray(unbundled),
+                BundledIds = new JsonArray()
+            };
+
+            return incomingEvent;
+        }
+
+        private JsonArray CreateJsonArray(IEnumerable<string> list)
+        {
+            var jsonArray = new JsonArray();
+            foreach (string value in list)
+            {
+                jsonArray.Add(value);
+            }
+
+            return jsonArray;
+        }
+    }
+}

Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs

@@ -60,7 +60,8 @@ public override void Configure(Analytics analytics)
         {
             base.Configure(analytics);
 
-            // TODO: Add DestinationMetadata enrichment plugin
+            // Add DestinationMetadata enrichment plugin
+            Add(new DestinationMetadataPlugin());
 
             _pipeline = new EventPipeline(
                     analytics,

Analytics-CSharp/Segment/Analytics/State.cs

@@ -93,9 +93,7 @@ public IState Reduce(IState state)
                 {
                     // Check if the settings have this destination
                     Settings settings = systemState._settings;
-                    JsonObject integrations = settings.Integrations;
-                    integrations[_key] = true;
-                    settings.Integrations = integrations;
+                    settings.Integrations[_key] = true;
 
                     result = new System(systemState._configuration, settings, systemState._running);
                 }

Analytics-CSharp/Segment/Analytics/Timeline.cs

@@ -34,7 +34,7 @@ internal RawEvent Process(RawEvent incomingEvent)
             // Enrichment is like middleware, a chance to update the event across the board before going to destinations.
             RawEvent enrichmentResult = ApplyPlugins(PluginType.Enrichment, beforeResult);
 
-            // Make sure not to update the events during this next cycle. Since each destination may want different 
+            // Make sure not to update the events during this next cycle. Since each destination may want different
             // data than other destinations we don't want them conflicting and changing what a real result should be
             ApplyPlugins(PluginType.Destination, enrichmentResult);
 
@@ -155,15 +155,11 @@ internal void Add(Plugin plugin)
         {
             _plugins.Add(plugin);
 
-            Analytics analytics = plugin.Analytics;
-            analytics.AnalyticsScope.Launch(analytics.AnalyticsDispatcher, async () =>
+            Settings? settings = plugin.Analytics.Settings();
+            if (settings.HasValue)
             {
-                Settings? settings = await plugin.Analytics.SettingsAsync();
-                if (settings.HasValue)
-                {
-                    plugin.Update(settings.Value, UpdateType.Initial);
-                }
-            });
+                plugin.Update(settings.Value, UpdateType.Initial);
+            }
         }
 
         internal void Remove(Plugin plugin) => _plugins.RemoveAll(tempPlugin => tempPlugin == plugin);

Analytics-CSharp/Segment/Analytics/Types.cs

@@ -3,6 +3,13 @@
 
 namespace Segment.Analytics
 {
+    public class DestinationMetadata
+    {
+        public JsonArray Bundled { get; set; }
+        public JsonArray Unbundled { get; set; }
+        public JsonArray BundledIds { get; set; }
+    }
+
     public abstract class RawEvent
     {
         public virtual string Type { get; set; }
@@ -17,6 +24,8 @@ public abstract class RawEvent
 
         public JsonArray Metrics { get; set; }
 
+        public DestinationMetadata _metadata { get; set; }
+
         internal void ApplyRawEventData(RawEvent rawEvent)
         {
             AnonymousId = rawEvent.AnonymousId;

Tests/Plugins/DestinationMetadataPluginTest.cs

@@ -0,0 +1,139 @@
+using Moq;
+using Segment.Analytics;
+using Segment.Analytics.Plugins;
+using Segment.Analytics.Utilities;
+using Segment.Serialization;
+using Tests.Utils;
+using Xunit;
+
+namespace Tests.Plugins
+{
+    public class DestinationMetadataPluginTest
+    {
+        private readonly Analytics _analytics;
+
+        private Settings _settings;
+
+        private DestinationMetadataPlugin _metadataPlugin;
+
+        public DestinationMetadataPluginTest()
+        {
+            _settings = JsonUtility.FromJson<Settings>(
+                "{\"integrations\":{\"Segment.io\":{\"apiKey\":\"1vNgUqwJeCHmqgI9S1sOm9UHCyfYqbaQ\"}},\"plan\":{},\"edgeFunction\":{}}");
+
+            var mockHttpClient = new Mock<HTTPClient>(null, null, null);
+            mockHttpClient
+                .Setup(httpClient => httpClient.Settings())
+                .ReturnsAsync(_settings);
+
+            var config = new Configuration(
+                writeKey: "123",
+                storageProvider: new DefaultStorageProvider("tests"),
+                autoAddSegmentDestination: false,
+                userSynchronizeDispatcher: true,
+                httpClientProvider: new MockHttpClientProvider(mockHttpClient)
+            );
+            _analytics = new Analytics(config);
+            _metadataPlugin = new DestinationMetadataPlugin();
+            _metadataPlugin.Configure(_analytics);
+            _metadataPlugin.Update(_settings, UpdateType.Initial);
+        }
+
+        [Fact]
+        public void TestBundled()
+        {
+            var a = new StubDestinationPlugin("A");
+            var b = new StubDestinationPlugin("B");
+            var c = new StubDestinationPlugin("C");
+            _analytics.Add(a);
+            _analytics.Add(b);
+            _analytics.Add(c);
+            _analytics.ManuallyEnableDestination(a);
+            _analytics.ManuallyEnableDestination(b);
+            _analytics.ManuallyEnableDestination(c);
+
+            var trackEvent = new TrackEvent("test", new JsonObject());
+            RawEvent actual = _metadataPlugin.Execute(trackEvent);
+
+            Assert.Equal(3, actual._metadata.Bundled.Count);
+            Assert.Equal(0, actual._metadata.Unbundled.Count);
+            Assert.Equal(0, actual._metadata.BundledIds.Count);
+        }
+
+        [Fact]
+        public void TestIntegrationNotInBundled()
+        {
+            _settings.Integrations.Add("a", "test");
+            _settings.Integrations.Add("b", "test");
+            _settings.Integrations.Add("c", "test");
+            _metadataPlugin.Update(_settings, UpdateType.Refresh);
+
+            var trackEvent = new TrackEvent("test", new JsonObject());
+            RawEvent actual = _metadataPlugin.Execute(trackEvent);
+
+            Assert.Equal(0, actual._metadata.Bundled.Count);
+            Assert.Equal(3, actual._metadata.Unbundled.Count);
+            Assert.Equal(0, actual._metadata.BundledIds.Count);
+        }
+
+        [Fact]
+        public void TestUnbundledIntegrations()
+        {
+            _metadataPlugin.Update(new Settings
+            {
+                Integrations = new JsonObject
+                {
+                    ["Segment.io"] = new JsonObject
+                    {
+                        ["unbundledIntegrations"] = new JsonArray
+                        {
+                            "a", "b", "c"
+                        }
+                    }
+                }
+            }, UpdateType.Refresh);
+
+            var trackEvent = new TrackEvent("test", new JsonObject());
+            RawEvent actual = _metadataPlugin.Execute(trackEvent);
+
+            Assert.Equal(0, actual._metadata.Bundled.Count);
+            Assert.Equal(3, actual._metadata.Unbundled.Count);
+            Assert.Equal(0, actual._metadata.BundledIds.Count);
+        }
+
+        [Fact]
+        public void TestCombination()
+        {
+            // bundled
+            var a = new StubDestinationPlugin("A");
+            _analytics.Add(a);
+            _analytics.ManuallyEnableDestination(a);
+
+
+            _metadataPlugin.Update(new Settings
+            {
+                Integrations = new JsonObject
+                {
+                    // IntegrationNotInBundled
+                    ["b"] = "test",
+                    ["c"] = "test",
+                    ["Segment.io"] = new JsonObject
+                    {
+                        ["unbundledIntegrations"] = new JsonArray
+                        {
+                            // UnbundledIntegrations
+                            "d", "e", "f"
+                        }
+                    }
+                }
+            }, UpdateType.Refresh);
+
+            var trackEvent = new TrackEvent("test", new JsonObject());
+            RawEvent actual = _metadataPlugin.Execute(trackEvent);
+
+            Assert.Equal(1, actual._metadata.Bundled.Count);
+            Assert.Equal(5, actual._metadata.Unbundled.Count);
+            Assert.Equal(0, actual._metadata.BundledIds.Count);
+        }
+    }
+}

Tests/Utils/Stubs.cs

@@ -45,6 +45,16 @@ public class StubEventPlugin : EventPlugin
         public override PluginType Type => PluginType.Before;
     }
 
+    public class StubDestinationPlugin : DestinationPlugin
+    {
+        public override string Key { get; }
+
+        public StubDestinationPlugin(string key)
+        {
+            Key = key;
+        }
+    }
+
     public class MockStorageProvider : IStorageProvider
     {
         public Mock<IStorage> Mock { get; set; }