diff --git a/CoarseTiingoNewsUniverseSelectionAlgorithm.cs b/CoarseTiingoNewsUniverseSelectionAlgorithm.cs new file mode 100644 index 0000000..cf9fe3f --- /dev/null +++ b/CoarseTiingoNewsUniverseSelectionAlgorithm.cs @@ -0,0 +1,120 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using QuantConnect.Interfaces; +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Algorithm; +using QuantConnect.Data; +using QuantConnect.DataSource; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Securities; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Example algorithm of a custom universe selection using coarse data and adding TiingoNews + /// If conditions are met will add the underlying and trade it + /// + public class CoarseTiingoNewsUniverseSelectionAlgorithm : QCAlgorithm + { + private const int NumberOfSymbols = 3; + private List _symbols; + + public override void Initialize() + { + SetStartDate(2014, 03, 24); + SetEndDate(2014, 04, 07); + + UniverseSettings.FillForward = false; + + AddUniverse(new CustomDataCoarseFundamentalUniverse(UniverseSettings, CoarseSelectionFunction)); + + _symbols = new List(); + } + + // sort the data by daily dollar volume and take the top 'NumberOfSymbols' + public IEnumerable CoarseSelectionFunction(IEnumerable coarse) + { + // sort descending by daily dollar volume + var sortedByDollarVolume = coarse.OrderByDescending(x => x.DollarVolume); + + // take the top entries from our sorted collection + var top = sortedByDollarVolume.Take(NumberOfSymbols); + + // we need to return only the symbol objects + return top.Select(x => QuantConnect.Symbol.CreateBase(typeof(TiingoNews), x.Symbol, x.Symbol.ID.Market)); + } + + public override void OnData(Slice data) + { + var articles = data.Get(); + + foreach (var kvp in articles) + { + var news = kvp.Value; + if (news.Title.IndexOf("Stocks Drop", 0, StringComparison.CurrentCultureIgnoreCase) != -1) + { + if (!Securities.ContainsKey(kvp.Key.Underlying)) + { + // add underlying we want to trade + AddSecurity(kvp.Key.Underlying); + _symbols.Add(kvp.Key.Underlying); + } + } + } + + foreach (var symbol in _symbols) + { + if (Securities[symbol].HasData) + { + SetHoldings(symbol, 1m / _symbols.Count); + } + } + } + + public override void OnSecuritiesChanged(SecurityChanges changes) + { + changes.FilterCustomSecurities = false; + Log($"{Time} {changes}"); + } + + private class CustomDataCoarseFundamentalUniverse : CoarseFundamentalUniverse + { + public CustomDataCoarseFundamentalUniverse(UniverseSettings universeSettings, Func, IEnumerable> selector) + : base(universeSettings, selector) + { } + + public override IEnumerable GetSubscriptionRequests(Security security, DateTime currentTimeUtc, DateTime maximumEndTimeUtc, + ISubscriptionDataConfigService subscriptionService) + { + var config = subscriptionService.Add( + typeof(TiingoNews), + security.Symbol, + UniverseSettings.Resolution, + UniverseSettings.FillForward, + UniverseSettings.ExtendedMarketHours, + dataNormalizationMode: UniverseSettings.DataNormalizationMode); + return new[]{new SubscriptionRequest(isUniverseSubscription: false, + universe: this, + security: security, + configuration: config, + startTimeUtc: currentTimeUtc, + endTimeUtc: maximumEndTimeUtc)}; + } + } + } +} diff --git a/CoarseTiingoNewsUniverseSelectionAlgorithm.py b/CoarseTiingoNewsUniverseSelectionAlgorithm.py new file mode 100644 index 0000000..4ae09e0 --- /dev/null +++ b/CoarseTiingoNewsUniverseSelectionAlgorithm.py @@ -0,0 +1,68 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from QuantConnect.DataSource import * + +### +### Example algorithm of a custom universe selection using coarse data and adding TiingoNews +### If conditions are met will add the underlying and trade it +### +class CoarseTiingoNewsUniverseSelectionAlgorithm(QCAlgorithm): + + def Initialize(self): + '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.''' + + self.SetStartDate(2014,3,24) + self.SetEndDate(2014,4,7) + + self.UniverseSettings.FillForward = False; + + self.__numberOfSymbols = 3 + + self.AddUniverse(CustomDataCoarseFundamentalUniverse(self.UniverseSettings, self.CoarseSelectionFunction)); + + self._symbols = [] + + # sort the data by daily dollar volume and take the top 'NumberOfSymbols' + def CoarseSelectionFunction(self, coarse): + # sort descending by daily dollar volume + sortedByDollarVolume = sorted(coarse, key=lambda x: x.DollarVolume, reverse=True) + + # return the symbol objects of the top entries from our sorted collection + return [ Symbol.CreateBase(TiingoNews, x.Symbol, x.Symbol.ID.Market) for x in sortedByDollarVolume[:self.__numberOfSymbols] ] + + def OnData(self, data): + articles = data.Get(TiingoNews) + + for kvp in articles: + news = kvp.Value + if "stocks drop" in news.Title.lower(): + if not self.Securities.ContainsKey(kvp.Key.Underlying): + # add underlying we want to trade + self.AddSecurity(kvp.Key.Underlying) + self._symbols.append(kvp.Key.Underlying) + + for symbol in self._symbols: + if self.Securities[symbol].HasData: + self.SetHoldings(symbol, 1.0 / len(self._symbols)) + + def OnSecuritiesChanged(self, changes): + changes.FilterCustomSecurities = False + self.Log(f"{self.Time} {changes}") + +class CustomDataCoarseFundamentalUniverse(CoarseFundamentalUniverse): + def GetSubscriptionRequests(self, security, currentTimeUtc, maximumEndTimeUtc, subscriptionService): + us = self.UniverseSettings + config = subscriptionService.Add(TiingoNews, security.Symbol, us.Resolution, us.FillForward, us.ExtendedMarketHours, True, False, False, us.DataNormalizationMode) + return [ SubscriptionRequest(False, self, security, config, currentTimeUtc, maximumEndTimeUtc) ] diff --git a/DataQueueHandlers/QuantConnect.DataSource.DataQueueHandlers.csproj b/DataQueueHandlers/QuantConnect.DataSource.DataQueueHandlers.csproj new file mode 100644 index 0000000..f789c04 --- /dev/null +++ b/DataQueueHandlers/QuantConnect.DataSource.DataQueueHandlers.csproj @@ -0,0 +1,22 @@ + + + + net5.0 + QuantConnect.DataSource.DataQueueHandlers + QuantConnect.DataSource.DataQueueHandlers.TiingoNews + + + + + + + + + + + + + + + + diff --git a/DataQueueHandlers/TiingoNewsDataQueueHandler.cs b/DataQueueHandlers/TiingoNewsDataQueueHandler.cs new file mode 100644 index 0000000..9f9b36e --- /dev/null +++ b/DataQueueHandlers/TiingoNewsDataQueueHandler.cs @@ -0,0 +1,193 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using QuantConnect.Configuration; +using QuantConnect.Data; +using QuantConnect.DataSource; +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Util; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using QuantConnect.Logging; +using QuantConnect.Packets; + +namespace QuantConnect.DataSource.DataQueueHandlers +{ + /// + /// Tiingo News Data queue handler + /// + /// + /// + public class TiingoNewsDataQueueHandler : IDataQueueHandler + { + private readonly List _symbolList = new List(); + private readonly bool _filterTicks; + private HashSet _emittedNews = new HashSet(); + private IDataAggregator _dataAggregator; + private readonly string _tiingoToken = Config.Get("tiingo-auth-token"); + private readonly TiingoNewsComparer _comparer = new TiingoNewsComparer(); + private readonly RealTimeScheduleEventService _realTimeSchedule = new RealTimeScheduleEventService(RealTimeProvider.Instance); + private readonly object _locker = new object(); + + /// + /// Initializes a new instance of the class. + /// + public TiingoNewsDataQueueHandler() + { + _realTimeSchedule.ScheduleEvent(TimeSpan.FromMinutes(1), DateTime.UtcNow); + _realTimeSchedule.NewEvent += GetLatestNews; + _dataAggregator = Composer.Instance.GetPart() ?? + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Data.Common.CustomDataAggregator")); + + _filterTicks = Config.GetBool("tiingo-news-filter-ticks", false); + } + + /// + /// Gets the latest news from Tiingo APi. + /// + /// The sender. + /// The instance containing the event data. + private void GetLatestNews(object sender, EventArgs e) + { + var url = $"https://api.tiingo.com/tiingo/news?token={_tiingoToken}&sortBy=crawlDate".ToStringInvariant(); + + try + { + string content; + using (var client = new WebClient()) + { + content = client.DownloadString(url); + } + + var utcNow = DateTime.UtcNow; + var lastHourNews = JsonConvert.DeserializeObject>(content, new TiingoNewsJsonConverter()) + .Where(n => utcNow - n.CrawlDate <= TimeSpan.FromHours(1)); + + _emittedNews = new HashSet(_emittedNews.Where(n => utcNow - n.CrawlDate < TimeSpan.FromHours(3)), _comparer); + + lock (_locker) + { + if (_filterTicks) + { + lastHourNews = lastHourNews.Where(n => n.Symbols.Intersect(_symbolList).Any()); + } + + foreach (var tiingoNews in lastHourNews.Where(n => _emittedNews.Add(n))) + foreach (var tiingoNewsSymbol in tiingoNews.Symbols) + { + if (_filterTicks && !_symbolList.Contains(tiingoNewsSymbol)) continue; + tiingoNews.Symbol = Symbol.CreateBase(typeof(Data.Custom.Tiingo.TiingoNews), tiingoNewsSymbol, Market.USA); + tiingoNews.Time = tiingoNews.CrawlDate; + _dataAggregator.Update(tiingoNews); + } + } + } + catch (Exception exception) + { + Log.Error(exception); + } + _realTimeSchedule.ScheduleEvent(TimeSpan.FromMinutes(1), DateTime.UtcNow); + } + + /// + /// Adds the specified symbols to the subscription + /// + /// defines the parameters to subscribe to a data feed + /// handler to be fired on new data available + /// The new enumerator for this subscription request + public IEnumerator Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + { + lock (_locker) + { + if (!_symbolList.Contains(dataConfig.Symbol)) + { + _symbolList.Add(dataConfig.Symbol); + } + } + return _dataAggregator.Add(dataConfig, newDataAvailableHandler); + } + + /// + /// Removes the specified configuration + /// + /// The data config to remove + public void Unsubscribe(SubscriptionDataConfig dataConfig) + { + lock (_locker) + { + _symbolList.Remove(dataConfig.Symbol); + } + _dataAggregator.Remove(dataConfig); + } + + /// + /// Sets the job we're subscribing for + /// + /// Job we're subscribing for + public void SetJob(LiveNodePacket job) + { + } + + /// + /// Returns whether the data provider is connected + /// + /// True if the data provider is connected + public bool IsConnected => true; + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + /// + public void Dispose() + { + _realTimeSchedule.NewEvent -= GetLatestNews; + _realTimeSchedule.DisposeSafely(); + } + } + + /// + /// IEqualityComparer implementation for Tiingo news data. + /// + /// + internal class TiingoNewsComparer : IEqualityComparer + { + /// + /// Check equality. + /// + /// The this one. + /// Another one. + /// + public bool Equals(Data.Custom.Tiingo.TiingoNews thisOne, Data.Custom.Tiingo.TiingoNews anotherOne) + { + return thisOne?.ArticleID == anotherOne?.ArticleID; + } + + /// + /// Returns a hash code for this instance. + /// + /// The object. + /// + /// A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table. + /// + public int GetHashCode(Data.Custom.Tiingo.TiingoNews obj) + { + return obj.ArticleID.GetHashCode(); + } + } +} \ No newline at end of file diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj index d08186e..db8bb83 100644 --- a/QuantConnect.DataSource.csproj +++ b/QuantConnect.DataSource.csproj @@ -20,7 +20,17 @@ + + + + + + + + + + diff --git a/TiingoNewsAlgorithm.cs b/TiingoNewsAlgorithm.cs new file mode 100644 index 0000000..0cb5c24 --- /dev/null +++ b/TiingoNewsAlgorithm.cs @@ -0,0 +1,86 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Algorithm; +using QuantConnect.Data; +using QuantConnect.DataSource; + +namespace QuantConnect.DataLibrary.Tests +{ + /// + /// Look for positive and negative words in the news article description + /// and trade based on the sum of the sentiment + /// + public class TiingoNewsAlgorithm : QCAlgorithm + { + private Symbol _tiingoSymbol; + + // Predefine a dictionary of words with scores to scan for in the description + // of the Tiingo news article + private readonly Dictionary _words = new Dictionary() + { + {"bad", -0.5}, {"good", 0.5}, + { "negative", -0.5}, {"great", 0.5}, + {"growth", 0.5}, {"fail", -0.5}, + {"failed", -0.5}, {"success", 0.5}, + {"nailed", 0.5}, {"beat", 0.5}, + {"missed", -0.5} + }; + + public override void Initialize() + { + SetStartDate(2019, 6, 10); + SetEndDate(2019, 10, 3); + SetCash(100000); + + var aapl = AddEquity("AAPL", Resolution.Hour).Symbol; + _tiingoSymbol = AddData(aapl).Symbol; + + // Request underlying equity data + var ibm = AddEquity("IBM", Resolution.Minute).Symbol; + // Add news data for the underlying IBM asset + var news = AddData(ibm).Symbol; + // Request 60 days of history with the TiingoNews IBM Custom Data Symbol. + var history = History(news, 60, Resolution.Daily); + + // Count the number of items we get from our history request + Debug($"We got {history.Count()} items from our history request"); + } + + public override void OnData(Slice data) + { + //Confirm that the data is in the collection + if (!data.ContainsKey(_tiingoSymbol)) return; + + // Gets the first piece of data from the Slice + var article = data.Get(_tiingoSymbol); + + // Article descriptions come in all caps. Lower and split by word + var descriptionWords = article.Description.ToLowerInvariant().Split(' '); + + // Take the intersection of predefined words and the words in the + // description to get a list of matching words + var intersection = _words.Keys.Intersect(descriptionWords); + + // Get the sum of the article's sentiment, and go long or short + // depending if it's a positive or negative description + var sentiment = intersection.Select(x => _words[x]).Sum(); + + SetHoldings(article.Symbol.Underlying, sentiment); + } + } +} \ No newline at end of file diff --git a/TiingoNewsAlgorithm.py b/TiingoNewsAlgorithm.py new file mode 100644 index 0000000..1c794d6 --- /dev/null +++ b/TiingoNewsAlgorithm.py @@ -0,0 +1,70 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from QuantConnect.DataSource import * + +### +### Look for positive and negative words in the news article description +### and trade based on the sum of the sentiment +### +class TiingoNewsAlgorithm(QCAlgorithm): + + def Initialize(self): + # Predefine a dictionary of words with scores to scan for in the description + # of the Tiingo news article + self.words = { + "bad": -0.5, "good": 0.5, + "negative": -0.5, "great": 0.5, + "growth": 0.5, "fail": -0.5, + "failed": -0.5, "success": 0.5, "nailed": 0.5, + "beat": 0.5, "missed": -0.5, + } + + self.SetStartDate(2019, 6, 10) + self.SetEndDate(2019, 10, 3) + self.SetCash(100000) + + aapl = self.AddEquity("AAPL", Resolution.Hour).Symbol + self.aaplCustom = self.AddData(TiingoNews, aapl).Symbol + + # Request underlying equity data. + ibm = self.AddEquity("IBM", Resolution.Minute).Symbol + # Add news data for the underlying IBM asset + news = self.AddData(TiingoNews, ibm).Symbol + # Request 60 days of history with the TiingoNews IBM Custom Data Symbol + history = self.History(TiingoNews, news, 60, Resolution.Daily) + + # Count the number of items we get from our history request + self.Debug(f"We got {len(history)} items from our history request") + + def OnData(self, data): + # Confirm that the data is in the collection + if not data.ContainsKey(self.aaplCustom): + return + + # Gets the data from the slice + article = data[self.aaplCustom] + + # Article descriptions come in all caps. Lower and split by word + descriptionWords = article.Description.lower().split(" ") + + # Take the intersection of predefined words and the words in the + # description to get a list of matching words + intersection = set(self.words.keys()).intersection(descriptionWords) + + # Get the sum of the article's sentiment, and go long or short + # depending if it's a positive or negative description + sentiment = sum([self.words[i] for i in intersection]) + + self.SetHoldings(article.Symbol.Underlying, sentiment) diff --git a/tests/TiingoNewsJsonConverterTests.cs b/tests/TiingoNewsJsonConverterTests.cs new file mode 100644 index 0000000..ce41f8a --- /dev/null +++ b/tests/TiingoNewsJsonConverterTests.cs @@ -0,0 +1,179 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using Newtonsoft.Json; +using NUnit.Framework; +using QuantConnect.DataSource; +using QuantConnect.DataProcessing; + +namespace QuantConnect.Tests.Common.Data.Custom +{ + [TestFixture] + public class TiingoNewsJsonConverterTests + { + [Test] + public void DeserializeCorrectly() + { + var content = @"[{ + ""source"":""source"", + ""crawlDate"":""2019-01-29T22:20:01.696871Z"", + ""description"":""description"", + ""url"":""url"", + ""publishedDate"":""2019-01-29T22:17:00Z"", + ""tags"":[ ""tag1"", ""tag2""], + ""tickers"":[""aapl""], + ""id"":1, + ""title"":""title"", + ""time"":""2019-01-29T23:17:00Z"", +}, +{ + ""source"":""source"", + ""crawlDate"":""2019-01-29T22:20:01.696871Z"", + ""publishedDate"":""2019-01-29T22:20:01.696871Z"", + ""tickers"":[], + ""id"":2, + ""title"":""title"", + ""time"":""2019-01-29T23:20:01.696871Z"" +}]"; + var result = JsonConvert.DeserializeObject>(content, + new TiingoNewsJsonConverter(Symbols.SPY)); + + Assert.AreEqual("2", result[0].ArticleID); + Assert.AreEqual( + DateTime.Parse("2019-01-29T22:20:01.696871", CultureInfo.InvariantCulture), + result[0].CrawlDate); + Assert.AreEqual( + DateTime.Parse("2019-01-29T22:20:01.696871", CultureInfo.InvariantCulture), + result[0].PublishedDate); + Assert.AreEqual("title", result[0].Title); + Assert.AreEqual(new List(), result[0].Symbols); + Assert.AreEqual(new List(), result[0].Tags); + Assert.AreEqual(result[0].PublishedDate.Add(TiingoNewsConverter.HistoricalCrawlOffset), result[0].Time); + Assert.AreEqual(result[0].PublishedDate.Add(TiingoNewsConverter.HistoricalCrawlOffset), result[0].EndTime); + + Assert.AreEqual(2, result.Count); + + Assert.AreEqual("1", result[1].ArticleID); + Assert.AreEqual( + DateTime.Parse("2019-01-29T22:20:01.696871", CultureInfo.InvariantCulture), + result[1].CrawlDate); + Assert.AreEqual( + DateTime.Parse("2019-01-29T22:17:00", CultureInfo.InvariantCulture), + result[1].PublishedDate); + Assert.AreEqual("description", result[1].Description); + Assert.AreEqual("source", result[1].Source); + Assert.AreEqual(new List { "tag1", "tag2" }, result[1].Tags); + Assert.AreEqual(new List { Symbols.AAPL }, result[1].Symbols); + Assert.AreEqual("title", result[1].Title); + Assert.AreEqual("url", result[1].Url); + Assert.AreEqual(Symbols.SPY, result[1].Symbol); + Assert.AreEqual(result[1].PublishedDate.Add(TiingoNewsConverter.HistoricalCrawlOffset), result[1].Time); + Assert.AreEqual(result[1].PublishedDate.Add(TiingoNewsConverter.HistoricalCrawlOffset), result[1].EndTime); + } + + [TestCase(true)] + [TestCase(false)] + public void RespectsHistoricalCrawlOffset(bool liveMode) + { + var content = @"[{ + ""source"":""source"", + ""crawlDate"":""2019-01-29T22:20:01.696871Z"", + ""description"":""description"", + ""url"":""url"", + ""publishedDate"":""2018-01-29T22:17:00Z"", + ""tags"":[ ""tag1"", ""tag2""], + ""tickers"":[""aapl""], + ""id"":1, + ""title"":""title"","; + if (!liveMode) + { + // live mode does not have 'time', this is added by the converter + content += @" ""time"":""2018-01-29T23:17:00Z"""; + } + content += @"}]"; + + var result = JsonConvert.DeserializeObject>(content, + new TiingoNewsJsonConverter(Symbols.SPY)); + + if (liveMode) + { + Assert.AreEqual(result[0].CrawlDate, result[0].Time); + } + else + { + Assert.AreEqual(result[0].PublishedDate.Add(TiingoNewsConverter.HistoricalCrawlOffset), result[0].Time); + } + Assert.AreEqual(result[0].EndTime, result[0].Time); + } + + [Test] + public void SerializeRoundTrip() + { + var settings = new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.All }; + + var crawlDate = new DateTime(2020, 3, 19, 10, 0, 0); + var underlyingSymbol = Symbols.AAPL; + var symbol = Symbol.CreateBase(typeof(TiingoNews), underlyingSymbol, QuantConnect.Market.USA); + var symbolList = new List { underlyingSymbol }; + var tags = new List { "Stock", "Technology" }; + + var item = new TiingoNews + { + ArticleID = "123456", + Symbol = symbol, + Symbols = symbolList, + Tags = tags, + Title = "title", + CrawlDate = crawlDate, + Time = crawlDate + }; + + var serialized = JsonConvert.SerializeObject(item, settings); + var deserialized = JsonConvert.DeserializeObject(serialized, settings); + + Assert.AreEqual("123456", deserialized.ArticleID); + Assert.AreEqual(symbol, deserialized.Symbol); + Assert.AreEqual(symbolList, deserialized.Symbols); + Assert.AreEqual(tags, deserialized.Tags); + Assert.AreEqual("title", deserialized.Title); + Assert.AreEqual(crawlDate, deserialized.CrawlDate); + Assert.AreEqual(crawlDate, deserialized.Time); + Assert.AreEqual(crawlDate, deserialized.EndTime); + } + + [Test] + public void DoesNotFailIfTickerContainsSpace() + { + var content = @"[{ + ""source"":""source"", + ""crawlDate"":""2019-01-29T22:20:01.696871Z"", + ""description"":""description"", + ""url"":""url"", + ""publishedDate"":""2018-01-29T22:17:00Z"", + ""tags"":[ ""tag1"", ""tag2""], + ""tickers"":[""aapl"", ""iff 6"", ""abc|1"", ""zxc | 1""], + ""id"":1, + ""title"":""title"" +}]"; + + Assert.DoesNotThrow(()=>JsonConvert.DeserializeObject>(content, + new TiingoNewsJsonConverter(Symbols.SPY))); + + } + } +}