Migrate TiingoNews custom data references to DataSource repo

CoarseTiingoNewsUniverseSelectionAlgorithm.cs

@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Algorithm;
+using QuantConnect.Data;
+using QuantConnect.DataSource;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Securities;
+
+namespace QuantConnect.DataLibrary.Tests
+{
+    /// <summary>
+    /// Example algorithm of a custom universe selection using coarse data and adding TiingoNews
+    /// If conditions are met will add the underlying and trade it
+    /// </summary>
+    public class CoarseTiingoNewsUniverseSelectionAlgorithm : QCAlgorithm
+    {
+        private const int NumberOfSymbols = 3;
+        private List<Symbol> _symbols;
+
+        public override void Initialize()
+        {
+            SetStartDate(2014, 03, 24);
+            SetEndDate(2014, 04, 07);
+
+            UniverseSettings.FillForward = false;
+
+            AddUniverse(new CustomDataCoarseFundamentalUniverse(UniverseSettings, CoarseSelectionFunction));
+
+            _symbols = new List<Symbol>();
+        }
+
+        // sort the data by daily dollar volume and take the top 'NumberOfSymbols'
+        public IEnumerable<Symbol> CoarseSelectionFunction(IEnumerable<CoarseFundamental> coarse)
+        {
+            // sort descending by daily dollar volume
+            var sortedByDollarVolume = coarse.OrderByDescending(x => x.DollarVolume);
+
+            // take the top entries from our sorted collection
+            var top = sortedByDollarVolume.Take(NumberOfSymbols);
+
+            // we need to return only the symbol objects
+            return top.Select(x => QuantConnect.Symbol.CreateBase(typeof(TiingoNews), x.Symbol, x.Symbol.ID.Market));
+        }
+
+        public override void OnData(Slice data)
+        {
+            var articles = data.Get<TiingoNews>();
+
+            foreach (var kvp in articles)
+            {
+                var news = kvp.Value;
+                if (news.Title.IndexOf("Stocks Drop", 0, StringComparison.CurrentCultureIgnoreCase) != -1)
+                {
+                    if (!Securities.ContainsKey(kvp.Key.Underlying))
+                    {
+                        // add underlying we want to trade
+                        AddSecurity(kvp.Key.Underlying);
+                        _symbols.Add(kvp.Key.Underlying);
+                    }
+                }
+            }
+
+            foreach (var symbol in _symbols)
+            {
+                if (Securities[symbol].HasData)
+                {
+                    SetHoldings(symbol, 1m / _symbols.Count);
+                }
+            }
+        }
+
+        public override void OnSecuritiesChanged(SecurityChanges changes)
+        {
+            changes.FilterCustomSecurities = false;
+            Log($"{Time} {changes}");
+        }
+
+        private class CustomDataCoarseFundamentalUniverse : CoarseFundamentalUniverse
+        {
+            public CustomDataCoarseFundamentalUniverse(UniverseSettings universeSettings, Func<IEnumerable<CoarseFundamental>, IEnumerable<Symbol>> selector)
+                : base(universeSettings, selector)
+            { }
+
+            public override IEnumerable<SubscriptionRequest> GetSubscriptionRequests(Security security, DateTime currentTimeUtc, DateTime maximumEndTimeUtc,
+                ISubscriptionDataConfigService subscriptionService)
+            {
+                var config = subscriptionService.Add(
+                    typeof(TiingoNews),
+                    security.Symbol,
+                    UniverseSettings.Resolution,
+                    UniverseSettings.FillForward,
+                    UniverseSettings.ExtendedMarketHours,
+                    dataNormalizationMode: UniverseSettings.DataNormalizationMode);
+                return new[]{new SubscriptionRequest(isUniverseSubscription: false,
+                    universe: this,
+                    security: security,
+                    configuration: config,
+                    startTimeUtc: currentTimeUtc,
+                    endTimeUtc: maximumEndTimeUtc)};
+            }
+        }
+    }
+}

CoarseTiingoNewsUniverseSelectionAlgorithm.py

@@ -0,0 +1,68 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+from QuantConnect.DataSource import *
+
+### <summary>
+### Example algorithm of a custom universe selection using coarse data and adding TiingoNews
+### If conditions are met will add the underlying and trade it
+### </summary>
+class CoarseTiingoNewsUniverseSelectionAlgorithm(QCAlgorithm):
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetStartDate(2014,3,24)
+        self.SetEndDate(2014,4,7)
+
+        self.UniverseSettings.FillForward = False;
+
+        self.__numberOfSymbols = 3
+
+        self.AddUniverse(CustomDataCoarseFundamentalUniverse(self.UniverseSettings, self.CoarseSelectionFunction));
+
+        self._symbols = []
+
+    # sort the data by daily dollar volume and take the top 'NumberOfSymbols'
+    def CoarseSelectionFunction(self, coarse):
+        # sort descending by daily dollar volume
+        sortedByDollarVolume = sorted(coarse, key=lambda x: x.DollarVolume, reverse=True)
+
+        # return the symbol objects of the top entries from our sorted collection
+        return [ Symbol.CreateBase(TiingoNews, x.Symbol, x.Symbol.ID.Market) for x in sortedByDollarVolume[:self.__numberOfSymbols] ]
+
+    def OnData(self, data):
+        articles = data.Get(TiingoNews)
+
+        for kvp in articles:
+            news = kvp.Value
+            if "stocks drop" in news.Title.lower():
+                if not self.Securities.ContainsKey(kvp.Key.Underlying):
+                    # add underlying we want to trade
+                    self.AddSecurity(kvp.Key.Underlying)
+                    self._symbols.append(kvp.Key.Underlying)
+
+        for symbol in self._symbols:
+            if self.Securities[symbol].HasData:
+                self.SetHoldings(symbol, 1.0 / len(self._symbols))
+
+    def OnSecuritiesChanged(self,  changes):
+        changes.FilterCustomSecurities = False
+        self.Log(f"{self.Time} {changes}")
+
+class CustomDataCoarseFundamentalUniverse(CoarseFundamentalUniverse):
+    def GetSubscriptionRequests(self, security, currentTimeUtc, maximumEndTimeUtc, subscriptionService):
+        us = self.UniverseSettings
+        config = subscriptionService.Add(TiingoNews, security.Symbol, us.Resolution, us.FillForward, us.ExtendedMarketHours, True, False, False, us.DataNormalizationMode)
+        return [ SubscriptionRequest(False, self, security, config, currentTimeUtc, maximumEndTimeUtc) ]

DataQueueHandlers/QuantConnect.DataSource.DataQueueHandlers.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <RootNamespace>QuantConnect.DataSource.DataQueueHandlers</RootNamespace>
+    <AssemblyName>QuantConnect.DataSource.DataQueueHandlers.TiingoNews</AssemblyName>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="QuantConnect.Common" Version="2.5.11800" />
+    <PackageReference Include="QuantConnect.Compression" Version="2.5.11800" />
+    <PackageReference Include="QuantConnect.Lean.Engine" Version="2.5.11800" />
+    <PackageReference Include="protobuf-net" Version="3.0.29" />
+    <PackageReference Include="protobuf-net.Core" Version="3.0.29" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\QuantConnect.DataSource.csproj" />
+  </ItemGroup>
+
+</Project>