26 using System.Collections.Generic;
28 using System.Collections.Concurrent;
42 private bool _customDataDownloadError;
43 private readonly ConcurrentDictionary<Symbol, Symbol> _marketHoursWarning =
new();
54 var dataDownloaderConfig =
Config.
Get(
"data-downloader");
55 if (!
string.IsNullOrEmpty(dataDownloaderConfig))
61 throw new ArgumentException(
"DownloaderDataProvider(): requires 'data-downloader' to be set with a valid type name");
70 _dataDownloader = dataDownloader;
78 public override Stream
Fetch(
string key)
82 if (
LeanData.
TryParsePath(key, out var symbol, out var date, out var resolution, out var tickType, out var dataType))
84 if (symbol.SecurityType == SecurityType.Base)
86 if (!_customDataDownloadError)
88 _customDataDownloadError = true;
90 Log.Trace($
"DownloaderDataProvider.Get(): custom data is not supported, requested: {symbol}");
98 entry = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
103 if (_marketHoursWarning.TryAdd(symbol, symbol))
106 Log.Trace($
"DownloaderDataProvider.Get(): failed to find market hours for {symbol}, skipping");
113 var exchangeTimeZone = entry.ExchangeHours.TimeZone;
114 DateTime startTimeUtc;
117 var endTimeUtcLimit = DateTime.UtcNow.Date.AddDays(-1);
121 startTimeUtc = date.ConvertToUtc(dataTimeZone);
123 endTimeUtc = date.AddDays(1).ConvertToUtc(dataTimeZone);
124 if (endTimeUtc > endTimeUtcLimit)
133 endTimeUtc = endTimeUtcLimit;
140 startTimeUtc = new DateTime(2009, 1, 1);
142 else if (symbol.SecurityType.IsOption() && symbol.SecurityType !=
SecurityType.FutureOption)
145 startTimeUtc = new DateTime(date.Year, 1, 1);
146 endTimeUtc = startTimeUtc.AddYears(1);
150 startTimeUtc = symbol.ID.Date;
153 catch (InvalidOperationException)
160 startTimeUtc = Time.Start;
163 if (endTimeUtc > endTimeUtcLimit)
165 endTimeUtc = endTimeUtcLimit;
172 var getParams =
new DataDownloaderGetParameters(symbol, resolution, startTimeUtc, endTimeUtc, tickType);
174 var downloaderDataParameters = getParams.GetDataDownloaderParameterForAllMappedSymbols(_mapFileProvider, exchangeTimeZone);
176 var downloadedData =
GetDownloadedData(downloaderDataParameters, symbol, exchangeTimeZone, dataTimeZone, dataType);
178 foreach (var dataPerSymbol
in downloadedData)
182 writer =
new LeanDataWriter(resolution, symbol, Globals.DataFolder, tickType, mapSymbol:
true, dataCacheProvider: _dataCacheProvider);
185 writer.
Write(dataPerSymbol);
207 IEnumerable<DataDownloaderGetParameters> downloaderDataParameters,
209 DateTimeZone exchangeTimeZone,
210 DateTimeZone dataTimeZone,
213 if (downloaderDataParameters.IsNullOrEmpty())
215 throw new ArgumentException($
"{nameof(DownloaderDataProvider)}.{nameof(GetDownloadedData)}: DataDownloaderGetParameters are empty or equal to null.");
218 foreach (var downloaderDataParameter
in downloaderDataParameters)
220 var downloadedData = _dataDownloader.Get(downloaderDataParameter);
222 if (downloadedData ==
null)
228 var groupedData = FilterAndGroupDownloadDataBySymbol(
234 downloaderDataParameter.StartUtc,
235 downloaderDataParameter.EndUtc);
237 foreach (var data
in groupedData)
252 return DiskSynchronizer.Execute(key, () =>
254 var baseStream = base.Fetch(key);
255 if (baseStream !=
null)
257 var result =
new MemoryStream();
258 baseStream.CopyTo(result);
259 baseStream.Dispose();
269 return base.Fetch(key);
281 || filePath.Contains(
"fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains(
"fundamental", StringComparison.InvariantCultureIgnoreCase)
282 || filePath.Contains(
"map_files", StringComparison.InvariantCultureIgnoreCase)
283 || filePath.Contains(
"factor_files", StringComparison.InvariantCultureIgnoreCase)
284 || filePath.Contains(
"margins", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains(
"future", StringComparison.InvariantCultureIgnoreCase))
291 return !File.Exists(filePath) || filePath.IsOutOfDate();
308 IEnumerable<BaseData> downloadData,
311 DateTimeZone exchangeTimeZone,
312 DateTimeZone dataTimeZone,
313 DateTime downloaderStartTimeUtc,
314 DateTime downloaderEndTimeUtc)
316 var startDateTimeInExchangeTimeZone = downloaderStartTimeUtc.ConvertFromUtc(exchangeTimeZone);
317 var endDateTimeInExchangeTimeZone = downloaderEndTimeUtc.ConvertFromUtc(exchangeTimeZone);
323 if (baseData.Time < startDateTimeInExchangeTimeZone || baseData.Time > endDateTimeInExchangeTimeZone)
331 baseData.Time = baseData.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
332 baseData.EndTime = baseData.EndTime.ConvertTo(exchangeTimeZone, dataTimeZone);
338 .GroupBy(baseData => baseData.Symbol);