Lean  $LEAN_TAG$
ApiDataProvider.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  *
15 */
16 
17 using System;
18 using System.IO;
19 using System.Linq;
20 using System.Threading;
21 using QuantConnect.Api;
22 using QuantConnect.Util;
23 using QuantConnect.Logging;
25 using System.Collections.Generic;
27 
29 {
30  /// <summary>
31  /// An instance of the <see cref="IDataProvider"/> that will download and update data files as needed via QC's Api.
32  /// </summary>
34  {
35  private decimal _purchaseLimit = Config.GetValue("data-purchase-limit", decimal.MaxValue); //QCC
36 
37  private readonly HashSet<SecurityType> _unsupportedSecurityType;
38  private readonly DataPricesList _dataPrices;
39  private readonly IApi _api;
40  private readonly bool _subscribedToIndiaEquityMapAndFactorFiles;
41  private readonly bool _subscribedToUsaEquityMapAndFactorFiles;
42  private readonly bool _subscribedToFutureMapAndFactorFiles;
43  private volatile bool _invalidSecurityTypeLog;
44 
45  /// <summary>
46  /// Initialize a new instance of the <see cref="ApiDataProvider"/>
47  /// </summary>
48  public ApiDataProvider()
49  {
50  _unsupportedSecurityType = new HashSet<SecurityType> { SecurityType.Future, SecurityType.FutureOption, SecurityType.Index, SecurityType.IndexOption };
51 
52  _api = Composer.Instance.GetPart<IApi>();
53 
54  // If we have no value for organization get account preferred
55  if (string.IsNullOrEmpty(Globals.OrganizationID))
56  {
57  var account = _api.ReadAccount();
58  Globals.OrganizationID = account?.OrganizationId;
59  Log.Trace($"ApiDataProvider(): Will use organization Id '{Globals.OrganizationID}'.");
60  }
61 
62  // Read in data prices and organization details
63  _dataPrices = _api.ReadDataPrices(Globals.OrganizationID);
64  var organization = _api.ReadOrganization(Globals.OrganizationID);
65 
66  foreach (var productItem in organization.Products.Where(x => x.Type == ProductType.Data).SelectMany(product => product.Items))
67  {
68  if (productItem.Id == 37)
69  {
70  // Determine if the user is subscribed to Equity map and factor files (Data product Id 37)
71  _subscribedToUsaEquityMapAndFactorFiles = true;
72  }
73  else if (productItem.Id == 137)
74  {
75  // Determine if the user is subscribed to Future map and factor files (Data product Id 137)
76  _subscribedToFutureMapAndFactorFiles = true;
77  }
78  else if (productItem.Id == 172)
79  {
80  // Determine if the user is subscribed to India map and factor files (Data product Id 172)
81  _subscribedToIndiaEquityMapAndFactorFiles = true;
82  }
83  }
84 
85  // Verify user has agreed to data provider agreements
86  if (organization.DataAgreement.Signed)
87  {
88  //Log Agreement Highlights
89  Log.Trace("ApiDataProvider(): Data Terms of Use has been signed. \r\n" +
90  $" Find full agreement at: {_dataPrices.AgreementUrl} \r\n" +
91  "==========================================================================\r\n" +
92  $"CLI API Access Agreement: On {organization.DataAgreement.SignedTime:d} You Agreed:\r\n" +
93  " - Display or distribution of data obtained through CLI API Access is not permitted. \r\n" +
94  " - Data and Third Party Data obtained via CLI API Access can only be used for individual or internal employee's use.\r\n" +
95  " - Data is provided in LEAN format can not be manipulated for transmission or use in other applications. \r\n" +
96  " - QuantConnect is not liable for the quality of data received and is not responsible for trading losses. \r\n" +
97  "==========================================================================");
98  Thread.Sleep(TimeSpan.FromSeconds(3));
99  }
100  else
101  {
102  // Log URL to go accept terms
103  throw new InvalidOperationException($"ApiDataProvider(): Must agree to terms at {_dataPrices.AgreementUrl}, before using the ApiDataProvider");
104  }
105 
106  // Verify we have the balance to maintain our purchase limit, if not adjust it to meet our balance
107  var balance = organization.Credit.Balance;
108  if (balance < _purchaseLimit)
109  {
110  if (_purchaseLimit != decimal.MaxValue)
111  {
112  Log.Error("ApiDataProvider(): Purchase limit is greater than balance." +
113  $" Setting purchase limit to balance : {balance}");
114  }
115  _purchaseLimit = balance;
116  }
117  }
118 
119  /// <summary>
120  /// Retrieves data to be used in an algorithm.
121  /// If file does not exist, an attempt is made to download them from the api
122  /// </summary>
123  /// <param name="key">File path representing where the data requested</param>
124  /// <returns>A <see cref="Stream"/> of the data requested</returns>
125  public override Stream Fetch(string key)
126  {
127  return DownloadOnce(key, s =>
128  {
129  // Verify we have enough credit to handle this
130  var pricePath = Api.Api.FormatPathForDataRequest(key);
131  var price = _dataPrices.GetPrice(pricePath);
132 
133  // No price found
134  if (price == -1)
135  {
136  throw new ArgumentException($"ApiDataProvider.Fetch(): No price found for {pricePath}");
137  }
138 
139  if (_purchaseLimit < price)
140  {
141  throw new ArgumentException($"ApiDataProvider.Fetch(): Cost {price} for {pricePath} data exceeds remaining purchase limit: {_purchaseLimit}");
142  }
143 
144  if (DownloadData(key))
145  {
146  // Update our purchase limit.
147  _purchaseLimit -= price;
148  }
149  });
150  }
151 
152  /// <summary>
153  /// Main filter to determine if this file needs to be downloaded
154  /// </summary>
155  /// <param name="filePath">File we are looking at</param>
156  /// <returns>True if should download</returns>
157  protected override bool NeedToDownload(string filePath)
158  {
159  // Ignore null
160  if (filePath == null)
161  {
162  return false;
163  }
164 
165  // Some security types can't be downloaded, lets attempt to extract that information
166  if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType, out var market) && _unsupportedSecurityType.Contains(securityType))
167  {
168  // we do support future auxiliary data (map and factor files)
169  if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
170  {
171  if (!_invalidSecurityTypeLog)
172  {
173  // let's log this once. Will still use any existing data on disk
174  _invalidSecurityTypeLog = true;
175  Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
176  }
177  return false;
178  }
179  }
180 
181  if (securityType == SecurityType.Equity && filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase))
182  {
183  // Ignore fine fundamental data requests
184  return false;
185  }
186 
187  // Only download if it doesn't exist or is out of date.
188  // Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
189  var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
190 
191  if (shouldDownload)
192  {
193  if (securityType == SecurityType.Future)
194  {
195  if (!_subscribedToFutureMapAndFactorFiles)
196  {
197  throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
198  "to download Future auxiliary data from QuantConnect. " +
199  "Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
200  }
201  }
202  // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
203  else if (!_subscribedToUsaEquityMapAndFactorFiles && market.Equals(Market.USA, StringComparison.InvariantCultureIgnoreCase)
204  && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
205  {
206  throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
207  "to download Equity data from QuantConnect. " +
208  "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
209  }
210  else if (!_subscribedToIndiaEquityMapAndFactorFiles && market.Equals(Market.India, StringComparison.InvariantCultureIgnoreCase)
211  && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
212  {
213  throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
214  "to download India data from QuantConnect. " +
215  "Please visit https://www.quantconnect.com/datasets/truedata-india-equity-security-master for details.");
216  }
217  }
218 
219  return shouldDownload;
220  }
221 
222  /// <summary>
223  /// Attempt to download data using the Api for and return a FileStream of that data.
224  /// </summary>
225  /// <param name="filePath">The path to store the file</param>
226  /// <returns>A FileStream of the data</returns>
227  protected virtual bool DownloadData(string filePath)
228  {
229  if (Log.DebuggingEnabled)
230  {
231  Log.Debug($"ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for {filePath}.");
232  }
233 
234  if (_api.DownloadData(filePath, Globals.OrganizationID))
235  {
236  Log.Trace($"ApiDataProvider.Fetch(): Successfully retrieved data for {filePath}.");
237  return true;
238  }
239  // Failed to download; _api.DownloadData() will post error
240  return false;
241  }
242 
243  /// <summary>
244  /// Helper method to determine if this filepath is auxiliary data
245  /// </summary>
246  /// <param name="filepath">The target file path</param>
247  /// <returns>True if this file is of auxiliary data</returns>
248  private static bool IsAuxiliaryData(string filepath)
249  {
250  return filepath.Contains("map_files", StringComparison.InvariantCulture)
251  || filepath.Contains("factor_files", StringComparison.InvariantCulture)
252  || filepath.Contains("fundamental", StringComparison.InvariantCulture)
253  || filepath.Contains("shortable", StringComparison.InvariantCulture);
254  }
255  }
256 }