Use C# generic history api for custom Python universe

Algorithm/QCAlgorithm.History.cs

@@ -1141,9 +1141,15 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
                     var isCustom = Extensions.IsCustomDataType(symbol, type);
                     var entry = MarketHoursDatabase.GetEntry(symbol, new[] { type });
 
+                    // If the type is PythonData, we want to check what types are actually associated with the symbol
+                    if (!_customPythonDataTypes.TryGetValue(symbol, out var dataType))
+                    {
+                        dataType = type;
+                    }
+
                     // we were giving a specific type let's fetch it
                     return new[] { new SubscriptionDataConfig(
-                        type,
+                        dataType,
                         symbol,
                         resolution.Value,
                         entry.DataTimeZone,
@@ -1152,7 +1158,7 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
                         UniverseSettings.ExtendedMarketHours,
                         true,
                         isCustom,
-                        LeanData.GetCommonTickTypeForCommonDataTypes(type, symbol.SecurityType),
+                        LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType),
                         true,
                         UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType))};
                 }

Algorithm/QCAlgorithm.Python.cs

@@ -40,6 +40,8 @@ public partial class QCAlgorithm
     {
         private readonly Dictionary<IntPtr, PythonIndicator> _pythonIndicators = new Dictionary<IntPtr, PythonIndicator>();
 
+        private Dictionary<Symbol, Type> _customPythonDataTypes = new Dictionary<Symbol, Type>();
+
         /// <summary>
         /// PandasConverter for this Algorithm
         /// </summary>
@@ -264,6 +266,12 @@ private Security AddDataImpl(Type dataType, Symbol symbol, Resolution? resolutio
             var alias = symbol.ID.Symbol;
             SymbolCache.Set(alias, symbol);
 
+            // Cache the data type for custom python data to keep reference to the actual created type
+            if (dataType.IsAssignableTo(typeof(PythonData)))
+            {
+                CacheCustomPythonDataType(symbol, dataType);
+            }
+
             if (timeZone != null)
             {
                 // user set time zone
@@ -1887,5 +1895,10 @@ private PyObject TryCleanupCollectionDataFrame(Type dataType, PyObject history)
             }
             return history;
         }
+
+        private void CacheCustomPythonDataType(Symbol symbol, Type dataType)
+        {
+            _customPythonDataTypes[symbol] = dataType;
+        }
     }
 }

Algorithm/QCAlgorithm.Universe.cs

@@ -697,6 +697,7 @@ private SubscriptionDataConfig GetCustomUniverseConfiguration(Type dataType, str
                 }
                 // same as 'AddData<>' 'T' type will be treated as custom/base data type with always open market hours
                 universeSymbol = QuantConnect.Symbol.Create(name, SecurityType.Base, market, baseDataType: dataType);
+                CacheCustomPythonDataType(universeSymbol, dataType);
             }
             var marketHoursDbEntry = MarketHoursDatabase.GetEntry(universeSymbol, new[] { dataType });
             var dataTimeZone = marketHoursDbEntry.DataTimeZone;

Tests/Algorithm/AlgorithmHistoryTests.cs

@@ -3455,6 +3455,106 @@ def get_universe_history(algorithm, flatten):
             }
         }
 
+        [Test]
+        public void PythonCustomUniverseHistoryCanBeFetchedUsingCSharpApi()
+        {
+            var algorithm = GetAlgorithm(new DateTime(2018, 6, 1));
+
+            using (Py.GIL())
+            {
+                var testModule = PyModule.FromString("PythonCustomUniverseHistoryCanBeFetchedUsingCSharpApi",
+                    @"
+from AlgorithmImports import *
+
+
+class StockDataSource(PythonData):
+
+    def get_source(self, config: SubscriptionDataConfig, date: datetime, is_live: bool) -> SubscriptionDataSource:
+        source = ""../../TestData/daily-stock-picker-backtest.csv""
+        return SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv)
+
+    def reader(self, config: SubscriptionDataConfig, line: str, date: datetime, is_live: bool) -> BaseData:
+        if not (line.strip() and line[0].isdigit()): return None
+
+        stocks = StockDataSource()
+        stocks.symbol = config.symbol
+
+        try:
+            csv = line.split(',')
+            stocks.time = datetime.strptime(csv[0], ""%Y%m%d"")
+            stocks.end_time = stocks.time + timedelta(days=1)
+            stocks[""Symbols""] = csv[1:]
+
+        except ValueError:
+            # Do nothing
+            return None
+
+        return stocks
+
+def universe_selector(data):
+        return [x.symbol for x in data]
+
+def add_universe(algorithm):
+    return algorithm.add_universe(StockDataSource, ""universe-stock-data-source"", Resolution.DAILY, universe_selector)
+
+def get_history(algorithm, universe):
+    return list(algorithm.history[StockDataSource](universe.symbol, datetime(2018, 1, 1), datetime(2018, 6, 1), Resolution.DAILY))
+");
+
+                dynamic getUniverse = testModule.GetAttr("add_universe");
+                dynamic getHistory = testModule.GetAttr("get_history");
+
+                var universe = getUniverse(algorithm);
+                var history = getHistory(algorithm, universe).As<List<PythonData>>() as List<PythonData>;
+                Assert.IsNotEmpty(history);
+            }
+        }
+
+        [Test]
+        public void PythonCustomDataHistoryCanBeFetchedUsingCSharpApi()
+        {
+            var algorithm = GetAlgorithm(new DateTime(2013, 10, 8));
+
+            using (Py.GIL())
+            {
+                var testModule = PyModule.FromString("PythonCustomDataHistoryCanBeFetchedUsingCSharpApi",
+                    @"
+from AlgorithmImports import *
+from QuantConnect.Tests import *
+
+class MyCustomDataType(PythonData):
+
+    def get_source(self, config: SubscriptionDataConfig, date: datetime, is_live: bool) -> SubscriptionDataSource:
+        fileName = LeanData.GenerateZipFileName(Symbols.SPY, date, Resolution.MINUTE, config.TickType)
+        source = f'{Globals.DataFolder}equity/usa/minute/spy/{fileName}'
+        return SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv)
+
+    def reader(self, config: SubscriptionDataConfig, line: str, date: datetime, is_live: bool) -> BaseData:
+        data = line.split(',')
+        result = MyCustomDataType()
+        result.DataType = MarketDataType.Base
+        result.Symbol = config.Symbol
+        result.Time = date + timedelta(milliseconds=int(data[0]))
+        result.Value = 1
+
+        return result
+
+def add_data(algorithm):
+    return algorithm.add_data(MyCustomDataType, ""MyCustomDataType"", Resolution.DAILY)
+
+def get_history(algorithm, security):
+    return list(algorithm.history[MyCustomDataType](security.symbol, datetime(2013, 10, 7), datetime(2013, 10, 8), Resolution.MINUTE))
+");
+
+                dynamic getCustomSecurity = testModule.GetAttr("add_data");
+                dynamic getHistory = testModule.GetAttr("get_history");
+
+                var security = getCustomSecurity(algorithm);
+                var history = getHistory(algorithm, security).As<List<PythonData>>() as List<PythonData>;
+                Assert.IsNotEmpty(history);
+            }
+        }
+
         public class CustomUniverseData : BaseDataCollection
         {
             public decimal Weight { get; private set; }