Indices support (#5)

* Add support for indices in symbol mapper

* Add support for streaming indices data

* Add support for index historical data

* Minor fixes

* Address minor peer review

QuantConnect.Polygon.Tests/PolygonDataDownloaderTests.cs

@@ -57,6 +57,30 @@ public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TimeSp
             PolygonHistoryTests.AssertHistoricalDataResults(data, resolution);
         }
 
+        private static TestCaseData[] IndexHistoricalDataTestCases => PolygonHistoryTests.IndexHistoricalDataTestCases;
+
+        [TestCaseSource(nameof(IndexHistoricalDataTestCases))]
+        [Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
+        public void DownloadsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpy)
+        {
+            var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
+            var request = PolygonHistoryTests.CreateHistoryRequest(symbol, resolution, tickType, period);
+
+            var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);
+            var data = _downloader.Get(parameters).ToList();
+
+            Log.Trace("Data points retrieved: " + data.Count);
+
+            if (shouldBeEmpy)
+            {
+                Assert.That(data, Is.Empty);
+            }
+            else
+            {
+                PolygonHistoryTests.AssertHistoricalDataResults(data, resolution);
+            }
+        }
+
         [Test]
         [Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
         public void DownloadsDataFromCanonicalOptionSymbol()

QuantConnect.Polygon.Tests/PolygonDataQueueHandlerBaseTests.cs

@@ -128,7 +128,7 @@ public void CanSubscribeAndUnsubscribe()
         [TestCase(Resolution.Hour, 3)]
         [Explicit("Tests are dependent on network and take long. " +
             "Also, this test will only pass if the subscribed securities are liquid enough to get data in the test run time.")]
-        public void StreamsDataForDifferentResolutions(Resolution resolution, int period)
+        public virtual void StreamsDataForDifferentResolutions(Resolution resolution, int period)
         {
             using var polygon = new PolygonDataQueueHandler(ApiKey);
 

QuantConnect.Polygon.Tests/PolygonDataQueueHandlerIndicesTests.cs

@@ -0,0 +1,100 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+using QuantConnect.Logging;
+using QuantConnect.Polygon;
+using QuantConnect.Util;
+
+namespace QuantConnect.Tests.Polygon
+{
+    [TestFixture]
+    [Explicit("Tests are dependent on network and take long")]
+    public class PolygonDataQueueHandlerIndicesTests : PolygonDataQueueHandlerBaseTests
+    {
+        // Overriding because tick data is not supported for indices
+        [TestCase(Resolution.Second, 15)]
+        [TestCase(Resolution.Minute, 3)]
+        [TestCase(Resolution.Hour, 3)]
+        [Explicit("Tests are dependent on network and take long. " +
+            "Also, this test will only pass if the subscribed securities are liquid enough to get data in the test run time.")]
+        public override void StreamsDataForDifferentResolutions(Resolution resolution, int period)
+        {
+            base.StreamsDataForDifferentResolutions(resolution, period);
+        }
+
+        [Test]
+        [Explicit("Tests are dependent on network and take long. " +
+            "Also, this test will only pass if the subscribed securities are liquid enough to get data in the test run time.")]
+        public void TickDataIsNotSupportedForStreaming()
+        {
+            using var polygon = new PolygonDataQueueHandler(ApiKey);
+
+            var configs = GetConfigs(Resolution.Tick);
+            var receivedData = new List<BaseData>();
+
+            foreach (var config in configs)
+            {
+                polygon.Subscribe(config, (sender, args) =>
+                {
+                    var dataPoint = (BaseData)((NewDataAvailableEventArgs)args).DataPoint;
+                    Log.Trace($"{dataPoint}. Time span: {dataPoint.Time} - {dataPoint.EndTime}");
+
+                    receivedData.Add(dataPoint);
+                });
+            }
+
+            Thread.Sleep(TimeSpan.FromSeconds(10));
+
+            Log.Trace("Unsubscribing symbols");
+            foreach (var config in configs)
+            {
+                polygon.Unsubscribe(config);
+            }
+
+            Assert.That(receivedData, Is.Empty);
+        }
+
+        /// <summary>
+        /// The subscription data configs to be used in the tests. At least 2 configs are required
+        /// </summary>
+        /// <remarks>
+        /// In order to successfully run the tests, valid contracts should be used. Update them
+        /// </remarks>
+        protected override List<SubscriptionDataConfig> GetConfigs(Resolution resolution = Resolution.Second)
+        {
+            return new[] { "SPX", "VIX", "DJI", "IXIC" }
+                .Select(ticker =>
+                {
+                    var symbol = Symbol.Create(ticker, SecurityType.Index, Market.USA);
+                    return new[]
+                    {
+                        GetSubscriptionDataConfig<TradeBar>(symbol, resolution),
+                        GetSubscriptionDataConfig<QuoteBar>(symbol, resolution),
+                    };
+                })
+                .SelectMany(x => x)
+                .ToList();
+        }
+    }
+}

QuantConnect.Polygon.Tests/PolygonHistoryTests.cs

@@ -120,6 +120,49 @@ internal static void AssertHistoricalDataResults(List<BaseData> history, Resolut
             }
         }
 
+        internal static TestCaseData[] IndexHistoricalDataTestCases
+        {
+            get
+            {
+                return new[]
+                {
+                    // Trades
+                    new TestCaseData(Resolution.Tick, TimeSpan.FromMinutes(5), TickType.Trade, true),   // Tick data is not available for indexes
+                    new TestCaseData(Resolution.Second, TimeSpan.FromMinutes(30), TickType.Trade, false),
+                    new TestCaseData(Resolution.Minute, TimeSpan.FromDays(15), TickType.Trade, false),
+                    new TestCaseData(Resolution.Hour, TimeSpan.FromDays(180), TickType.Trade, false),
+                    new TestCaseData(Resolution.Daily, TimeSpan.FromDays(3650), TickType.Trade, false),
+
+                    // Quotes: quote data is not available for indexes
+                    new TestCaseData(Resolution.Tick, TimeSpan.FromMinutes(5), TickType.Quote, true),
+                    new TestCaseData(Resolution.Second, TimeSpan.FromMinutes(5), TickType.Quote, true),
+                    new TestCaseData(Resolution.Minute, TimeSpan.FromMinutes(5), TickType.Quote, true),
+                    new TestCaseData(Resolution.Hour, TimeSpan.FromMinutes(5), TickType.Quote, true),
+                    new TestCaseData(Resolution.Daily, TimeSpan.FromMinutes(5), TickType.Quote, true),
+                };
+            }
+        }
+
+        [TestCaseSource(nameof(IndexHistoricalDataTestCases))]
+        [Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
+        public void GetsIndexHistoricalData(Resolution resolution, TimeSpan period, TickType tickType, bool shouldBeEmpty)
+        {
+            var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
+            var requests = new List<HistoryRequest> { CreateHistoryRequest(symbol, resolution, tickType, period) };
+            var history = _historyProvider.GetHistory(requests, TimeZones.Utc).ToList();
+
+            Log.Trace("Data points retrieved: " + history.Count);
+
+            if (shouldBeEmpty)
+            {
+                Assert.That(history, Is.Empty);
+            }
+            else
+            {
+                AssertHistoricalDataResults(history.Select(x => x.AllData).SelectMany(x => x).ToList(), resolution, _historyProvider.DataPointCount);
+            }
+        }
+
         [Test]
         [Explicit("This tests require a Polygon.io api key, requires internet and are long.")]
         public void GetsSameBarCountForDifferentResponseLimits()
@@ -165,7 +208,6 @@ public void GetsSameBarCountForDifferentResponseLimits()
             new TestCaseData(Symbols.USDJPY, Resolution.Minute, TickType.Trade),
             new TestCaseData(Symbols.BTCUSD, Resolution.Minute, TickType.Trade),
             new TestCaseData(Symbols.DE10YBEUR, Resolution.Minute, TickType.Trade),
-            new TestCaseData(Symbols.SPX, Resolution.Minute, TickType.Trade),
             new TestCaseData(Symbols.Future_ESZ18_Dec2018, Resolution.Minute, TickType.Trade),
 
             // Supported security type and resolution, unsupported tick type

QuantConnect.Polygon.Tests/PolygonSymbolMapperTests.cs

@@ -94,5 +94,39 @@ public void ConvertsWeeklyIndexOptionSymbolWithoutParameters()
             Assert.That(symbol.ID.OptionStyle, Is.EqualTo(OptionStyle.American));
             Assert.That(symbol.ID.StrikePrice, Is.EqualTo(strike));
         }
+
+        [Test]
+        public void ConvertsLeanIndexSymbolToPolygon()
+        {
+            var mapper = new PolygonSymbolMapper();
+            var symbol = Symbol.Create("SPX", SecurityType.Index, Market.USA);
+            var polygonSymbol = mapper.GetBrokerageSymbol(symbol);
+            Assert.That(polygonSymbol, Is.EqualTo("I:SPX"));
+        }
+
+        [Test]
+        public void ConvertsPolygonIndexSymbolToLean()
+        {
+            var mapper = new PolygonSymbolMapper();
+            var symbol = mapper.GetLeanSymbol("I:SPX");
+            Assert.That(symbol, Is.EqualTo(Symbol.Create("SPX", SecurityType.Index, Market.USA)));
+        }
+
+        [Test]
+        public void ConvertsLeanEquitySymbolToPolygon()
+        {
+            var mapper = new PolygonSymbolMapper();
+            var symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
+            var polygonSymbol = mapper.GetBrokerageSymbol(symbol);
+            Assert.That(polygonSymbol, Is.EqualTo("AAPL"));
+        }
+
+        [Test]
+        public void ConvertsPolygonEquitySymbolToLean()
+        {
+            var mapper = new PolygonSymbolMapper();
+            var symbol = mapper.GetLeanSymbol("AAPL");
+            Assert.That(symbol, Is.EqualTo(Symbol.Create("AAPL", SecurityType.Equity, Market.USA)));
+        }
     }
 }

QuantConnect.Polygon/PolygonDataQueueHandler.cs

@@ -48,7 +48,8 @@ public partial class PolygonDataQueueHandler : IDataQueueHandler
         {
             SecurityType.Equity,
             SecurityType.Option,
-            SecurityType.IndexOption
+            SecurityType.IndexOption,
+            SecurityType.Index,
         });
 
         private readonly string _apiKey;

QuantConnect.Polygon/PolygonOptionChainProvider.cs

@@ -65,11 +65,10 @@ public IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
             }
 
             var underlying = symbol.SecurityType.IsOption() ? symbol.Underlying : symbol;
-            var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(underlying);
             var optionsSecurityType = underlying.SecurityType == SecurityType.Index ? SecurityType.IndexOption : SecurityType.Option;
 
             var request = new RestRequest("/v3/reference/options/contracts", Method.GET);
-            request.AddQueryParameter("underlying_ticker", brokerageSymbol);
+            request.AddQueryParameter("underlying_ticker", underlying.ID.Symbol);
             request.AddQueryParameter("as_of", date.ToStringInvariant("yyyy-MM-dd"));
             request.AddQueryParameter("limit", "1000");
 

QuantConnect.Polygon/PolygonSymbolMapper.cs

@@ -44,16 +44,20 @@ public string GetBrokerageSymbol(Symbol symbol)
             {
                 if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol))
                 {
+                    var ticker = symbol.Value.Replace(" ", "");
                     switch (symbol.SecurityType)
                     {
                         case SecurityType.Equity:
+                            brokerageSymbol = ticker;
+                            break;
+
                         case SecurityType.Index:
-                            brokerageSymbol = symbol.Value.Replace(" ", "");
+                            brokerageSymbol = $"I:{ticker}";
                             break;
 
                         case SecurityType.Option:
                         case SecurityType.IndexOption:
-                            brokerageSymbol = $"O:{symbol.Value.Replace(" ", "")}";
+                            brokerageSymbol = $"O:{ticker}";
                             break;
 
                         default:
@@ -123,14 +127,18 @@ public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, s
                             break;
 
                         case SecurityType.IndexOption:
-                            underlying ??= Symbol.Create(leanBaseSymbol?.Underlying.ID.Symbol, SecurityType.Index, market);
-                            leanSymbol = Symbol.CreateOption(underlying, leanBaseSymbol?.ID.Symbol, market, optionStyle, optionRight, strike, expirationDate);
+                            underlying ??= Symbol.Create(leanBaseSymbol.Underlying.ID.Symbol, SecurityType.Index, market);
+                            leanSymbol = Symbol.CreateOption(underlying, leanBaseSymbol.ID.Symbol, market, optionStyle, optionRight, strike, expirationDate);
                             break;
 
                         case SecurityType.Equity:
                             leanSymbol = Symbol.Create(brokerageSymbol, securityType, market);
                             break;
 
+                        case SecurityType.Index:
+                            leanSymbol = Symbol.Create(leanBaseSymbol.ID.Symbol, securityType, market);
+                            break;
+
                         default:
                             throw new Exception($"PolygonSymbolMapper.GetLeanSymbol(): unsupported security type: {securityType}");
                     }
@@ -172,6 +180,10 @@ private Symbol GetLeanSymbolInternal(string polygonSymbol)
             {
                 return GetLeanOptionSymbol(polygonSymbol);
             }
+            else if (polygonSymbol.StartsWith("I:"))
+            {
+                return GetLeanIndexSymbol(polygonSymbol);
+            }
 
             return GetLeanSymbol(polygonSymbol, SecurityType.Equity, Market.USA);
         }
@@ -195,10 +207,16 @@ private Symbol GetLeanOptionSymbol(string polygonSymbol)
             var underlying = IndexOptionSymbol.IsIndexOption(ticker)
                 ? Symbol.Create(IndexOptionSymbol.MapToUnderlying(ticker), SecurityType.Index, Market.USA)
                 : Symbol.Create(ticker, SecurityType.Equity, Market.USA);
-            var symbol = Symbol.CreateOption(underlying, ticker, Market.USA, OptionStyle.American, optionRight, strike, expirationDate);
-            _leanSymbolsCache[polygonSymbol] = symbol;
 
-            return symbol;
+            return Symbol.CreateOption(underlying, ticker, Market.USA, OptionStyle.American, optionRight, strike, expirationDate);
+        }
+
+        /// <summary>
+        /// Gets the Lean index symbol for the specified Polygon symbol
+        /// </summary>
+        private Symbol GetLeanIndexSymbol(string polygonSymbol)
+        {
+            return Symbol.Create(polygonSymbol.Substring(2), SecurityType.Index, Market.USA);
         }
     }
 }