Merge pull request QuantConnect#1 from QuantConnect/template-vendor-d…

…ownloader-converter-impl

Create template/skeleton data downloader/converter classes

DataLibrary/DataLibrary.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net5.0</TargetFramework>
+    <RootNamespace>QuantConnect.DataLibrary</RootNamespace>
   </PropertyGroup>
 
   <ItemGroup>

DataLibrary/MyCustomDataType.cs

@@ -22,7 +22,7 @@
 using QuantConnect.Data;
 using System.Collections.Generic;
 
-namespace DataLibrary
+namespace QuantConnect.DataLibrary
 {
     /// <summary>
     /// Example custom data type

DataProcessing/DataProcessing.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net5.0</TargetFramework>
+        <AssemblyName>process</AssemblyName>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="QuantConnect.Common" Version="2.5.11800" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\DataLibrary\DataLibrary.csproj" />
+    </ItemGroup>
+
+</Project>

DataProcessing/Program.cs

@@ -0,0 +1,78 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Configuration;
+using QuantConnect.Logging;
+using QuantConnect.Util;
+
+namespace QuantConnect.DataProcessing
+{
+    /// <summary>
+    /// Entrypoint for the data downloader/converter
+    /// </summary>
+    public class Program
+    {
+        /// <summary>
+        /// Entrypoint of the program
+        /// </summary>
+        /// <returns>Exit code. 0 equals successful, and any other value indicates the downloader/converter failed.</returns>
+        public static int Main()
+        {
+            // Get the configuration values required for your data downloader/converter.
+            // You will most likely be getting and setting up the values your downloader/converter need here.
+            var destinationDataFolderDirectory = Config.Get("temp-output-directory", "/temp-output-directory");
+            var apiKey = Config.Get($"{VendorDataDownloaderConverter.VendorName}-{VendorDataDownloaderConverter.VendorDataName}-api-key", null);
+
+            VendorDataDownloaderConverter instance;
+            try
+            {
+                // Pass in the values we got from the configuration into the downloader/converter.
+                instance = new VendorDataDownloaderConverter(destinationDataFolderDirectory, apiKey);
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"The downloader/converter for {VendorDataDownloaderConverter.VendorDataName} {VendorDataDownloaderConverter.VendorDataName} data failed to be constructed");
+                return 1;
+            }
+
+            // No need to edit anything below here for most use cases.
+            // The downloader/converter is ran and cleaned up for you safely here.
+            try
+            {
+                // Run the data downloader/converter.
+                var success = instance.Run();
+                if (!success)
+                {
+                    Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {VendorDataDownloaderConverter.VendorName} {VendorDataDownloaderConverter.VendorDataName} data");
+                    return 1;
+                }
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"The downloader/converter for {VendorDataDownloaderConverter.VendorDataName} {VendorDataDownloaderConverter.VendorDataName} data exited unexpectedly");
+                return 1;
+            }
+            finally
+            {
+                // Run cleanup of the downloader/converter once it has finished or crashed.
+                instance.DisposeSafely();
+            }
+
+            // The downloader/converter was successful
+            return 0;
+        }
+    }
+}

DataProcessing/VendorDataDownloaderConverter.cs

@@ -0,0 +1,180 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Logging;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using QuantConnect.DataLibrary;
+
+namespace QuantConnect.DataProcessing
+{
+    /// <summary>
+    /// Data downloader/converter class example
+    /// </summary>
+    public class VendorDataDownloaderConverter : IDisposable
+    {
+        /// <summary>
+        /// Name of the vendor. Only alphanumeric characters, all lowercase and no underscores/spaces. 
+        /// </summary>
+        public const string VendorName = "gaussgodel";
+
+        /// <summary>
+        /// Type of data that we are downloading/converting. Only alphanumeric characters, all lowercase and no underscores/spaces. 
+        /// </summary>
+        public const string VendorDataName = "flights";
+
+        /// <summary>
+        /// API key to download data with
+        /// </summary>
+        private readonly string _apiKey;
+
+        /// <summary>
+        /// Directory that data will be outputted to
+        /// </summary>
+        private readonly string _destinationDirectory;
+
+        /// <summary>
+        /// Has the class been cleaned up yet
+        /// </summary>
+        private bool _disposed;
+
+        /// <summary>
+        /// Creates a new instance of the data downloader/converter
+        /// </summary>
+        /// <param name="destinationDataFolder">The path to the data folder we want to save data to</param>
+        /// <param name="apiKey">The API key to download data with</param>
+        public VendorDataDownloaderConverter(string destinationDataFolder, string apiKey = null)
+        {
+            _apiKey = apiKey;
+            _destinationDirectory = Path.Combine(
+                destinationDataFolder,
+                "alternative",
+                VendorName,
+                VendorDataName);
+
+            // Create the directory ahead of time so that we don't get
+            // errors when trying to write to this output directory.
+            Directory.CreateDirectory(_destinationDirectory);
+        }
+
+        /// <summary>
+        /// Begins running the downloader/converter
+        /// </summary>
+        /// <returns>True if downloading/converting was successful, false otherwise</returns>
+        public bool Run()
+        {
+            try
+            {
+                // Your data downloading/processing code goes here. The lines below
+                // can be deleted since they are only meant to be an example.
+                // ================================================================
+                var underlying = Symbol.Create("SPY", SecurityType.Equity, Market.USA);
+                var symbol = Symbol.CreateBase(
+                    typeof(MyCustomDataType),
+                    underlying,
+                    Market.USA);
+
+                var lines = new[]
+                {
+                    "20131001,buy",
+                    "20131003,buy",
+                    "20131006,buy",
+                    "20131007,sell",
+                    "20131009,buy",
+                    "20131011,sell"
+                };
+
+                var instances = lines.Select(x => ParseLine(symbol, x));
+                var csvLines = instances.Select(x => ToCsvLine(x));
+
+                SaveContentsToFile(symbol, csvLines);
+            }
+            catch (Exception e)
+            {
+                Log.Error(e, $"Failed to download/convert {VendorName} {VendorDataName} data");
+                return false;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Example method to parse and create an instance from a line of CSV
+        /// </summary>
+        /// <param name="symbol">Symbol</param>
+        /// <param name="line">Line of raw data</param>
+        /// <returns>Instance of <see cref="MyCustomDataType"/></returns>
+        private MyCustomDataType ParseLine(Symbol symbol, string line)
+        {
+            var csv = line.Split(',');
+            return new MyCustomDataType
+            {
+                Time = Parse.DateTimeExact(csv[0], "yyyyMMdd"),
+                Symbol = symbol,
+
+                SomeCustomProperty = csv[1]
+            };
+        }
+
+        /// <summary>
+        /// Example method to convert an instance to a CSV line
+        /// </summary>
+        /// <param name="instance">Custom Data instance</param>
+        /// <returns>CSV line</returns>
+        private string ToCsvLine(MyCustomDataType instance)
+        {
+            return string.Join(",",
+                $"{instance.Time:yyyyMMdd}",
+                instance.SomeCustomProperty);
+        }
+
+        /// <summary>
+        /// Example method to save CSV lines to disk
+        /// </summary>
+        /// <param name="symbol">Symbol of the data</param>
+        /// <param name="csvLines">CSV lines to write</param>
+        private void SaveContentsToFile(Symbol symbol, IEnumerable<string> csvLines)
+        {
+            var ticker = symbol.Value.ToLowerInvariant();
+
+            var tempPath = new FileInfo(Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}-{ticker}.csv"));
+            var finalPath = Path.Combine(_destinationDirectory, $"{ticker}.csv");
+
+            File.WriteAllLines(tempPath.FullName, csvLines);
+            tempPath.MoveTo(finalPath, true);
+        }
+
+        /// <summary>
+        /// If you need to shut down things like database connections, threads, or other
+        /// resources that require manual shutdown, do it here. This will be called after
+        /// we're done with the <see cref="Run"/> method.
+        ///
+        /// You don't have to implement this if you don't need to cleanup resources after we're done downloading/processing.
+        /// </summary>
+        public void Dispose()
+        {
+            if (_disposed)
+            {
+                return;
+            }
+
+            // Your cleanup goes here. You don't have to implement it
+            // if you have nothing that needs to be cleaned up.
+            _disposed = true;
+        }
+    }
+}

MyCustomDataType.sln

@@ -7,6 +7,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataLibrary", "DataLibrary\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{46223B6F-6400-4C76-9066-441E8ACB6A21}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{9F1E4E69-1D72-4432-93FD-2257EE8FCE0A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -20,6 +22,10 @@ Global
 		{46223B6F-6400-4C76-9066-441E8ACB6A21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{46223B6F-6400-4C76-9066-441E8ACB6A21}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{46223B6F-6400-4C76-9066-441E8ACB6A21}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9F1E4E69-1D72-4432-93FD-2257EE8FCE0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9F1E4E69-1D72-4432-93FD-2257EE8FCE0A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9F1E4E69-1D72-4432-93FD-2257EE8FCE0A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9F1E4E69-1D72-4432-93FD-2257EE8FCE0A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Tests/CustomDataAlgorithm.cs

@@ -14,14 +14,12 @@
  *
 */
 
-using DataLibrary;
-using QuantConnect;
 using QuantConnect.Data;
 using QuantConnect.Util;
 using QuantConnect.Orders;
 using QuantConnect.Algorithm;
 
-namespace Tests
+namespace QuantConnect.DataLibrary.Tests
 {
     /// <summary>
     /// Example algorithm using the custom data type as a source of alpha

Tests/MyCustomDataTypeTests.cs

@@ -18,14 +18,12 @@
 using ProtoBuf;
 using System.IO;
 using System.Linq;
-using DataLibrary;
-using QuantConnect;
 using ProtoBuf.Meta;
 using Newtonsoft.Json;
 using NUnit.Framework;
 using QuantConnect.Data;
 
-namespace Tests
+namespace QuantConnect.DataLibrary.Tests
 {
     [TestFixture]
     public class MyCustomDataTypeTests

Tests/Tests.csproj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>net5.0</TargetFramework>
+    <RootNamespace>QuantConnect.DataLibrary.Tests</RootNamespace>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="protobuf-net" Version="3.0.29" />