20 using System.Threading;
25 using System.Collections.Generic;
35 private decimal _purchaseLimit =
Config.GetValue(
"data-purchase-limit", decimal.MaxValue);
37 private readonly HashSet<SecurityType> _unsupportedSecurityType;
39 private readonly
IApi _api;
40 private readonly
bool _subscribedToIndiaEquityMapAndFactorFiles;
41 private readonly
bool _subscribedToUsaEquityMapAndFactorFiles;
42 private readonly
bool _subscribedToFutureMapAndFactorFiles;
43 private volatile bool _invalidSecurityTypeLog;
59 Log.
Trace($
"ApiDataProvider(): Will use organization Id '{Globals.OrganizationID}'.");
66 foreach (var productItem
in organization.Products.Where(x => x.Type ==
ProductType.Data).SelectMany(product => product.Items))
68 if (productItem.Id == 37)
71 _subscribedToUsaEquityMapAndFactorFiles =
true;
73 else if (productItem.Id == 137)
76 _subscribedToFutureMapAndFactorFiles =
true;
78 else if (productItem.Id == 172)
81 _subscribedToIndiaEquityMapAndFactorFiles =
true;
86 if (organization.DataAgreement.Signed)
89 Log.
Trace(
"ApiDataProvider(): Data Terms of Use has been signed. \r\n" +
90 $
" Find full agreement at: {_dataPrices.AgreementUrl} \r\n" +
91 "==========================================================================\r\n" +
92 $
"CLI API Access Agreement: On {organization.DataAgreement.SignedTime:d} You Agreed:\r\n" +
93 " - Display or distribution of data obtained through CLI API Access is not permitted. \r\n" +
94 " - Data and Third Party Data obtained via CLI API Access can only be used for individual or internal employee's use.\r\n" +
95 " - Data is provided in LEAN format can not be manipulated for transmission or use in other applications. \r\n" +
96 " - QuantConnect is not liable for the quality of data received and is not responsible for trading losses. \r\n" +
97 "==========================================================================");
98 Thread.Sleep(TimeSpan.FromSeconds(3));
103 throw new InvalidOperationException($
"ApiDataProvider(): Must agree to terms at {_dataPrices.AgreementUrl}, before using the ApiDataProvider");
107 var balance = organization.Credit.Balance;
108 if (balance < _purchaseLimit)
110 if (_purchaseLimit != decimal.MaxValue)
112 Log.
Error(
"ApiDataProvider(): Purchase limit is greater than balance." +
113 $
" Setting purchase limit to balance : {balance}");
115 _purchaseLimit = balance;
125 public override Stream
Fetch(
string key)
130 var pricePath =
Api.
Api.FormatPathForDataRequest(key);
131 var price = _dataPrices.GetPrice(pricePath);
136 throw new ArgumentException($
"ApiDataProvider.Fetch(): No price found for {pricePath}");
139 if (_purchaseLimit < price)
141 throw new ArgumentException($
"ApiDataProvider.Fetch(): Cost {price} for {pricePath} data exceeds remaining purchase limit: {_purchaseLimit}");
147 _purchaseLimit -= price;
160 if (filePath ==
null)
169 if (securityType !=
SecurityType.Future || !IsAuxiliaryData(filePath))
171 if (!_invalidSecurityTypeLog)
174 _invalidSecurityTypeLog =
true;
175 Log.
Error($
"ApiDataProvider(): does not support security types: {string.Join(",
", _unsupportedSecurityType)}");
181 if (securityType ==
SecurityType.Equity && filePath.Contains(
"fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains(
"fundamental", StringComparison.InvariantCultureIgnoreCase))
189 var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
195 if (!_subscribedToFutureMapAndFactorFiles)
197 throw new ArgumentException(
"ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
198 "to download Future auxiliary data from QuantConnect. " +
199 "Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
203 else if (!_subscribedToUsaEquityMapAndFactorFiles && market.Equals(
Market.
USA, StringComparison.InvariantCultureIgnoreCase)
206 throw new ArgumentException(
"ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
207 "to download Equity data from QuantConnect. " +
208 "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
210 else if (!_subscribedToIndiaEquityMapAndFactorFiles && market.Equals(
Market.
India, StringComparison.InvariantCultureIgnoreCase)
213 throw new ArgumentException(
"ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
214 "to download India data from QuantConnect. " +
215 "Please visit https://www.quantconnect.com/datasets/truedata-india-equity-security-master for details.");
219 return shouldDownload;
231 Log.
Debug($
"ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for {filePath}.");
236 Log.
Trace($
"ApiDataProvider.Fetch(): Successfully retrieved data for {filePath}.");
248 private static bool IsAuxiliaryData(
string filepath)
250 return filepath.Contains(
"map_files", StringComparison.InvariantCulture)
251 || filepath.Contains(
"factor_files", StringComparison.InvariantCulture)
252 || filepath.Contains(
"fundamental", StringComparison.InvariantCulture)
253 || filepath.Contains(
"shortable", StringComparison.InvariantCulture);