diff --git a/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageDataQueueHandlerTests.cs b/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageDataQueueHandlerTests.cs index b45bab0..9cca8d8 100644 --- a/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageDataQueueHandlerTests.cs +++ b/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageDataQueueHandlerTests.cs @@ -22,6 +22,7 @@ using Microsoft.CodeAnalysis; using QuantConnect.Data.Market; using System.Collections.Generic; +using QuantConnect.Tests; namespace QuantConnect.CoinbaseBrokerage.Tests { @@ -39,6 +40,8 @@ private static IEnumerable TestParameters { get { + TestGlobals.Initialize(); + yield return new TestCaseData(BTCUSDC, Resolution.Tick); yield return new TestCaseData(BTCUSDC, Resolution.Second); yield return new TestCaseData(BTCUSDC, Resolution.Minute); diff --git a/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageHistoryProviderTests.cs b/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageHistoryProviderTests.cs index 1fd3f0f..3b4a436 100644 --- a/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageHistoryProviderTests.cs +++ b/QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageHistoryProviderTests.cs @@ -25,7 +25,6 @@ using System.Collections.Generic; using QuantConnect.Configuration; using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.CoinbaseBrokerage.Tests { @@ -33,60 +32,53 @@ namespace QuantConnect.CoinbaseBrokerage.Tests public class CoinbaseBrokerageHistoryProviderTests { [Test, TestCaseSource(nameof(TestParameters))] - public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool shouldBeEmpty) + public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool shouldBeEmpty, bool unsupported) { - var aggregator = new AggregationManager(); - var brokerage = new CoinbaseBrokerage( Config.Get("coinbase-url", "wss://advanced-trade-ws.coinbase.com"), Config.Get("coinbase-api-key"), Config.Get("coinbase-api-secret"), Config.Get("coinbase-rest-api", "https://api.coinbase.com"), null, - aggregator, + new AggregationManager(), null); - var historyProvider = new BrokerageHistoryProvider(); - historyProvider.SetBrokerage(brokerage); - historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, new DataPermissionManager(), null)); - var now = DateTime.UtcNow; - - var requests = new[] + var request = new HistoryRequest(now.Add(-period), + now, + typeof(TradeBar), + symbol, + resolution, + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + DateTimeZone.Utc, + resolution, + false, + false, + DataNormalizationMode.Adjusted, + tickType); + + var history = brokerage.GetHistory(request)?.ToList(); + + if (unsupported) { - new HistoryRequest(now.Add(-period), - now, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - DateTimeZone.Utc, - resolution, - false, - false, - DataNormalizationMode.Adjusted, - tickType) - }; - - var history = historyProvider.GetHistory(requests, TimeZones.Utc).ToList(); - - foreach (var slice in history) - { - var bar = slice.Bars[symbol]; - - Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}, V={bar.Volume}"); + Assert.IsNull(history); + return; } if (shouldBeEmpty) { - Assert.IsTrue(history.Count == 0); + Assert.IsEmpty(history); + return; } - else + + Assert.IsNotEmpty(history); + + foreach (var bar in history.Cast()) { - Assert.IsTrue(history.Count > 0); + Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}, V={bar.Volume}"); } - Log.Trace("Data points retrieved: " + historyProvider.DataPointCount); + Log.Trace("Data points retrieved: " + history.Count); } private static IEnumerable TestParameters @@ -98,29 +90,33 @@ private static IEnumerable TestParameters var BTCUSDC = Symbol.Create("BTCUSDC", SecurityType.Crypto, Market.Coinbase); // valid parameters - yield return new TestCaseData(BTCUSD, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(5), false); - yield return new TestCaseData(BTCUSD, Resolution.Minute, TickType.Trade, Time.OneHour, false); - yield return new TestCaseData(BTCUSD, Resolution.Hour, TickType.Trade, Time.OneDay, false); - yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), false); + yield return new TestCaseData(BTCUSD, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(5), false, false); + yield return new TestCaseData(BTCUSD, Resolution.Minute, TickType.Trade, Time.OneHour, false, false); + yield return new TestCaseData(BTCUSD, Resolution.Hour, TickType.Trade, Time.OneDay, false, false); + yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), false, false); - yield return new TestCaseData(BTCUSDC, Resolution.Minute, TickType.Trade, Time.OneHour, false); - yield return new TestCaseData(BTCUSDC, Resolution.Hour, TickType.Trade, Time.OneDay, false); + yield return new TestCaseData(BTCUSDC, Resolution.Minute, TickType.Trade, Time.OneHour, false, false); + yield return new TestCaseData(BTCUSDC, Resolution.Hour, TickType.Trade, Time.OneDay, false, false); - // quote tick type, no error, empty result - yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Quote, TimeSpan.FromDays(15), true); + // invalid period, no error, empty result + yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(-15), true, false); - // invalid resolution, no error, empty result - yield return new TestCaseData(BTCUSD, Resolution.Tick, TickType.Trade, TimeSpan.FromSeconds(15), true); - yield return new TestCaseData(BTCUSD, Resolution.Second, TickType.Trade, Time.OneMinute, true); + // quote tick type, null result + yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Quote, TimeSpan.FromDays(15), false, true); + yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.OpenInterest, TimeSpan.FromDays(15), false, true); - // invalid period, no error, empty result - yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(-15), true); + // invalid resolution, null result + yield return new TestCaseData(BTCUSD, Resolution.Tick, TickType.Trade, TimeSpan.FromSeconds(15), false, true); + yield return new TestCaseData(BTCUSD, Resolution.Second, TickType.Trade, Time.OneMinute, false, true); + + // invalid symbol, null result + yield return new TestCaseData(Symbol.Create("ABCXYZ", SecurityType.Crypto, Market.Coinbase), Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), false, true); - // invalid symbol, no error, empty result - yield return new TestCaseData(Symbol.Create("ABCXYZ", SecurityType.Crypto, Market.Coinbase), Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true); + // invalid security type, null result + yield return new TestCaseData(Symbols.EURGBP, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), false, true); - // invalid security type, no error, empty result - yield return new TestCaseData(Symbols.EURGBP, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true); + // invalid market, null result + yield return new TestCaseData(Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Binance), Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), false, true); } } } diff --git a/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloader.cs b/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloader.cs index 5f5b95a..2f635b8 100644 --- a/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloader.cs +++ b/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloader.cs @@ -43,13 +43,8 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa var endUtc = dataDownloaderGetParameters.EndUtc; var tickType = dataDownloaderGetParameters.TickType; - if (tickType != TickType.Trade) - { - return Enumerable.Empty(); - } - - var type = default(Type); - if(resolution == Resolution.Tick) + Type type; + if (resolution == Resolution.Tick) { type = typeof(Tick); } diff --git a/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloaderProgram.cs b/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloaderProgram.cs index 7e760cf..17f3867 100644 --- a/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloaderProgram.cs +++ b/QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloaderProgram.cs @@ -54,6 +54,10 @@ public static void CoinbaseDownloader(IList tickers, string resolution, // Download the data var symbolObject = Symbol.Create(ticker, SecurityType.Crypto, market); var data = downloader.Get(new DataDownloaderGetParameters(symbolObject, castResolution, fromDate, toDate)); + if (data == null) + { + continue; + } // Save the data var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory, TickType.Trade); diff --git a/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.HistoryProvider.cs b/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.HistoryProvider.cs index 759f98b..49b139c 100644 --- a/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.HistoryProvider.cs +++ b/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.HistoryProvider.cs @@ -20,6 +20,7 @@ using QuantConnect.Data.Market; using System.Collections.Generic; using QuantConnect.CoinbaseBrokerage.Models.Enums; +using System.Linq; namespace QuantConnect.CoinbaseBrokerage { @@ -31,7 +32,9 @@ public partial class CoinbaseBrokerage /// /// Prevent spam to external source /// - private bool _loggedCoinbaseSupportsOnlyTradeBars = false; + private bool _loggedCoinbaseSupportsOnlyTradeBars; + private bool _loggedUnsupportedAssetForHistory; + private bool _loggedUnsupportedResolutionForHistory; /// /// Gets the history for the requested security @@ -40,6 +43,16 @@ public partial class CoinbaseBrokerage /// An enumerable of bars covering the span specified in the request public override IEnumerable GetHistory(HistoryRequest request) { + if (!CanSubscribe(request.Symbol)) + { + if (!_loggedUnsupportedAssetForHistory) + { + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "UnsupportedAsset", + $"Unsupported asset: {request.Symbol.Value}, no history returned")); + } + return null; + } + // Coinbase API only allows us to support history requests for TickType.Trade if (request.TickType != TickType.Trade) { @@ -49,50 +62,37 @@ public override IEnumerable GetHistory(HistoryRequest request) _algorithm?.Debug($"Warning.{nameof(CoinbaseBrokerage)}: history provider only supports trade information, does not support quotes."); Log.Error($"{nameof(CoinbaseBrokerage)}.{nameof(GetHistory)}(): only supports TradeBars"); } - yield break; + return null; } - if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol)) - { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol", - $"Unknown symbol: {request.Symbol.Value}, no history returned")); - yield break; - } - - if (request.Symbol.SecurityType != SecurityType.Crypto) + if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType", - $"{request.Symbol.SecurityType} security type not supported, no history returned")); - yield break; + if (!_loggedUnsupportedResolutionForHistory) + { + _loggedUnsupportedResolutionForHistory = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", + $"{request.Resolution} resolution not supported, no history returned")); + } + return null; } if (request.StartTimeUtc >= request.EndTimeUtc) { OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange", "The history request start date must precede the end date, no history returned")); - yield break; - } - - if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) - { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", - $"{request.Resolution} resolution not supported, no history returned")); - yield break; + return Enumerable.Empty(); } Log.Debug($"{nameof(CoinbaseBrokerage)}.{nameof(GetHistory)}: Submitting request: {request.Symbol.Value}: {request.Resolution} {request.StartTimeUtc} UTC -> {request.EndTimeUtc} UTC"); - foreach (var tradeBar in GetHistoryFromCandles(request)) - { - yield return tradeBar; - } + return GetHistoryFromCandles(request); } /// /// Returns TradeBars from Coinbase candles (only for Minute/Hour/Daily resolutions) /// /// The history request instance - private IEnumerable GetHistoryFromCandles(HistoryRequest request) + private IEnumerable GetHistoryFromCandles(HistoryRequest request) { var productId = _symbolMapper.GetBrokerageSymbol(request.Symbol); var resolutionTimeSpan = request.Resolution.ToTimeSpan(); @@ -107,6 +107,7 @@ private IEnumerable GetHistoryFromCandles(HistoryRequest request) Resolution.Minute => CandleGranularity.OneMinute, Resolution.Hour => CandleGranularity.OneHour, Resolution.Daily => CandleGranularity.OneDay, + // This should never happen if the right checks are in place in the caller _ => throw new NotSupportedException($"The resolution {request.Resolution} is not supported.") }; diff --git a/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.cs b/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.cs index 9ce60ba..7b1d298 100644 --- a/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.cs +++ b/QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.cs @@ -370,7 +370,8 @@ protected virtual bool CanSubscribe(Symbol symbol) { return !symbol.Value.Contains("UNIVERSE") && symbol.SecurityType == SecurityType.Crypto && - symbol.ID.Market == MarketName; + symbol.ID.Market == MarketName && + _symbolMapper.IsKnownLeanSymbol(symbol); } #endregion