Lean  $LEAN_TAG$
LeanData.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using System;
17 using System.Collections.Generic;
18 using System.Globalization;
19 using System.IO;
20 using System.Linq;
21 using System.Runtime.CompilerServices;
22 using NodaTime;
23 using QuantConnect.Data;
29 using QuantConnect.Logging;
33 using static QuantConnect.StringExtensions;
34 
35 namespace QuantConnect.Util
36 {
37  /// <summary>
38  /// Provides methods for generating lean data file content
39  /// </summary>
40  public static class LeanData
41  {
42  private static readonly HashSet<Type> _strictDailyEndTimesDataTypes = new()
43  {
44  // the underlying could yield auxiliary data which we don't want to change
45  typeof(TradeBar), typeof(QuoteBar), typeof(ZipEntryName), typeof(BaseDataCollection), typeof(OpenInterest)
46  };
47 
48  /// <summary>
49  /// The different <see cref="SecurityType"/> used for data paths
50  /// </summary>
51  /// <remarks>This includes 'alternative'</remarks>
52  public static HashSet<string> SecurityTypeAsDataPath => Enum.GetNames(typeof(SecurityType))
53  .Select(x => x.ToLowerInvariant()).Union(new[] { "alternative" }).ToHashSet();
54 
55  /// <summary>
56  /// Converts the specified base data instance into a lean data file csv line.
57  /// This method takes into account the fake that base data instances typically
58  /// are time stamped in the exchange time zone, but need to be written to disk
59  /// in the data time zone.
60  /// </summary>
61  public static string GenerateLine(IBaseData data, Resolution resolution, DateTimeZone exchangeTimeZone, DateTimeZone dataTimeZone)
62  {
63  var clone = data.Clone();
64  clone.Time = data.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
65  return GenerateLine(clone, clone.Symbol.ID.SecurityType, resolution);
66  }
67 
68  /// <summary>
69  /// Helper method that will parse a given data line in search of an associated date time
70  /// </summary>
71  public static DateTime ParseTime(string line, DateTime date, Resolution resolution)
72  {
73  switch (resolution)
74  {
75  case Resolution.Tick:
76  case Resolution.Second:
77  case Resolution.Minute:
78  var index = line.IndexOf(',', StringComparison.InvariantCulture);
79  return date.AddTicks(Convert.ToInt64(10000 * decimal.Parse(line.AsSpan(0, index))));
80  case Resolution.Hour:
81  case Resolution.Daily:
82  return DateTime.ParseExact(line.AsSpan(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter, CultureInfo.InvariantCulture);
83  default:
84  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
85  }
86  }
87 
88  /// <summary>
89  /// Converts the specified base data instance into a lean data file csv line
90  /// </summary>
91  public static string GenerateLine(IBaseData data, SecurityType securityType, Resolution resolution)
92  {
93  var milliseconds = data.Time.TimeOfDay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);
94  var longTime = data.Time.ToStringInvariant(DateFormat.TwelveCharacter);
95 
96  switch (securityType)
97  {
98  case SecurityType.Equity:
99  switch (resolution)
100  {
101  case Resolution.Tick:
102  var tick = (Tick)data;
103  if (tick.TickType == TickType.Trade)
104  {
105  return ToCsv(milliseconds, Scale(tick.LastPrice), tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
106  }
107  if (tick.TickType == TickType.Quote)
108  {
109  return ToCsv(milliseconds, Scale(tick.BidPrice), tick.BidSize, Scale(tick.AskPrice), tick.AskSize, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
110  }
111  break;
112  case Resolution.Minute:
113  case Resolution.Second:
114  var tradeBar = data as TradeBar;
115  if (tradeBar != null)
116  {
117  return ToCsv(milliseconds, Scale(tradeBar.Open), Scale(tradeBar.High), Scale(tradeBar.Low), Scale(tradeBar.Close), tradeBar.Volume);
118  }
119  var quoteBar = data as QuoteBar;
120  if (quoteBar != null)
121  {
122  return ToCsv(milliseconds,
123  ToScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
124  ToScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
125  }
126  break;
127 
128  case Resolution.Hour:
129  case Resolution.Daily:
130  var bigTradeBar = data as TradeBar;
131  if (bigTradeBar != null)
132  {
133  return ToCsv(longTime, Scale(bigTradeBar.Open), Scale(bigTradeBar.High), Scale(bigTradeBar.Low), Scale(bigTradeBar.Close), bigTradeBar.Volume);
134  }
135  var bigQuoteBar = data as QuoteBar;
136  if (bigQuoteBar != null)
137  {
138  return ToCsv(longTime,
139  ToScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
140  ToScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
141  }
142  break;
143  }
144  break;
145 
146  case SecurityType.Crypto:
147  case SecurityType.CryptoFuture:
148  switch (resolution)
149  {
150  case Resolution.Tick:
151  var tick = data as Tick;
152  if (tick == null)
153  {
154  throw new ArgumentException($"{securityType} tick could not be created", nameof(data));
155  }
156  if (tick.TickType == TickType.Trade)
157  {
158  return ToCsv(milliseconds, tick.LastPrice, tick.Quantity, tick.Suspicious ? "1" : "0");
159  }
160  if (tick.TickType == TickType.Quote)
161  {
162  return ToCsv(milliseconds, tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.Suspicious ? "1" : "0");
163  }
164  throw new ArgumentException($"{securityType} tick could not be created");
165  case Resolution.Second:
166  case Resolution.Minute:
167  var quoteBar = data as QuoteBar;
168  if (quoteBar != null)
169  {
170  return ToCsv(milliseconds,
171  ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
172  ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
173  }
174  var tradeBar = data as TradeBar;
175  if (tradeBar != null)
176  {
177  return ToCsv(milliseconds, tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
178  }
179  throw new ArgumentException($"{securityType} minute/second bar could not be created", nameof(data));
180 
181  case Resolution.Hour:
182  case Resolution.Daily:
183  var bigQuoteBar = data as QuoteBar;
184  if (bigQuoteBar != null)
185  {
186  return ToCsv(longTime,
187  ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
188  ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
189  }
190  var bigTradeBar = data as TradeBar;
191  if (bigTradeBar != null)
192  {
193  return ToCsv(longTime,
194  bigTradeBar.Open,
195  bigTradeBar.High,
196  bigTradeBar.Low,
197  bigTradeBar.Close,
198  bigTradeBar.Volume);
199  }
200  throw new ArgumentException($"{securityType} hour/daily bar could not be created", nameof(data));
201  }
202  break;
203  case SecurityType.Forex:
204  case SecurityType.Cfd:
205  switch (resolution)
206  {
207  case Resolution.Tick:
208  var tick = data as Tick;
209  if (tick == null)
210  {
211  throw new ArgumentException("Expected data of type 'Tick'", nameof(data));
212  }
213  return ToCsv(milliseconds, tick.BidPrice, tick.AskPrice);
214 
215  case Resolution.Second:
216  case Resolution.Minute:
217  var bar = data as QuoteBar;
218  if (bar == null)
219  {
220  throw new ArgumentException("Expected data of type 'QuoteBar'", nameof(data));
221  }
222  return ToCsv(milliseconds,
223  ToNonScaledCsv(bar.Bid), bar.LastBidSize,
224  ToNonScaledCsv(bar.Ask), bar.LastAskSize);
225 
226  case Resolution.Hour:
227  case Resolution.Daily:
228  var bigBar = data as QuoteBar;
229  if (bigBar == null)
230  {
231  throw new ArgumentException("Expected data of type 'QuoteBar'", nameof(data));
232  }
233  return ToCsv(longTime,
234  ToNonScaledCsv(bigBar.Bid), bigBar.LastBidSize,
235  ToNonScaledCsv(bigBar.Ask), bigBar.LastAskSize);
236  }
237  break;
238 
239  case SecurityType.Index:
240  switch (resolution)
241  {
242  case Resolution.Tick:
243  var tick = (Tick)data;
244  return ToCsv(milliseconds, tick.LastPrice, tick.Quantity, string.Empty, string.Empty, "0");
245  case Resolution.Second:
246  case Resolution.Minute:
247  var bar = data as TradeBar;
248  if (bar == null)
249  {
250  throw new ArgumentException("Expected data of type 'TradeBar'", nameof(data));
251  }
252  return ToCsv(milliseconds, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume);
253  case Resolution.Hour:
254  case Resolution.Daily:
255  var bigTradeBar = data as TradeBar;
256  return ToCsv(longTime, bigTradeBar.Open, bigTradeBar.High, bigTradeBar.Low, bigTradeBar.Close, bigTradeBar.Volume);
257  }
258  break;
259 
260  case SecurityType.Option:
261  case SecurityType.IndexOption:
262  switch (resolution)
263  {
264  case Resolution.Tick:
265  var tick = (Tick)data;
266  if (tick.TickType == TickType.Trade)
267  {
268  return ToCsv(milliseconds,
269  Scale(tick.LastPrice), tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
270  }
271  if (tick.TickType == TickType.Quote)
272  {
273  return ToCsv(milliseconds,
274  Scale(tick.BidPrice), tick.BidSize, Scale(tick.AskPrice), tick.AskSize, tick.ExchangeCode, tick.Suspicious ? "1" : "0");
275  }
276  if (tick.TickType == TickType.OpenInterest)
277  {
278  return ToCsv(milliseconds, tick.Value);
279  }
280  break;
281 
282  case Resolution.Second:
283  case Resolution.Minute:
284  // option and future data can be quote or trade bars
285  var quoteBar = data as QuoteBar;
286  if (quoteBar != null)
287  {
288  return ToCsv(milliseconds,
289  ToScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
290  ToScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
291  }
292  var tradeBar = data as TradeBar;
293  if (tradeBar != null)
294  {
295  return ToCsv(milliseconds,
296  Scale(tradeBar.Open), Scale(tradeBar.High), Scale(tradeBar.Low), Scale(tradeBar.Close), tradeBar.Volume);
297  }
298  var openInterest = data as OpenInterest;
299  if (openInterest != null)
300  {
301  return ToCsv(milliseconds, openInterest.Value);
302  }
303  break;
304 
305  case Resolution.Hour:
306  case Resolution.Daily:
307  // option and future data can be quote or trade bars
308  var bigQuoteBar = data as QuoteBar;
309  if (bigQuoteBar != null)
310  {
311  return ToCsv(longTime,
312  ToScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
313  ToScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
314  }
315  var bigTradeBar = data as TradeBar;
316  if (bigTradeBar != null)
317  {
318  return ToCsv(longTime, ToScaledCsv(bigTradeBar), bigTradeBar.Volume);
319  }
320  var bigOpenInterest = data as OpenInterest;
321  if (bigOpenInterest != null)
322  {
323  return ToCsv(longTime, bigOpenInterest.Value);
324  }
325  break;
326 
327  default:
328  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
329  }
330  break;
331 
332  case SecurityType.FutureOption:
333  switch (resolution)
334  {
335  case Resolution.Tick:
336  var tick = (Tick)data;
337  if (tick.TickType == TickType.Trade)
338  {
339  return ToCsv(milliseconds,
340  tick.LastPrice, tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
341  }
342  if (tick.TickType == TickType.Quote)
343  {
344  return ToCsv(milliseconds,
345  tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.ExchangeCode, tick.Suspicious ? "1" : "0");
346  }
347  if (tick.TickType == TickType.OpenInterest)
348  {
349  return ToCsv(milliseconds, tick.Value);
350  }
351  break;
352 
353  case Resolution.Second:
354  case Resolution.Minute:
355  // option and future data can be quote or trade bars
356  var quoteBar = data as QuoteBar;
357  if (quoteBar != null)
358  {
359  return ToCsv(milliseconds,
360  ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
361  ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
362  }
363  var tradeBar = data as TradeBar;
364  if (tradeBar != null)
365  {
366  return ToCsv(milliseconds,
367  tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
368  }
369  var openInterest = data as OpenInterest;
370  if (openInterest != null)
371  {
372  return ToCsv(milliseconds, openInterest.Value);
373  }
374  break;
375 
376  case Resolution.Hour:
377  case Resolution.Daily:
378  // option and future data can be quote or trade bars
379  var bigQuoteBar = data as QuoteBar;
380  if (bigQuoteBar != null)
381  {
382  return ToCsv(longTime,
383  ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
384  ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
385  }
386  var bigTradeBar = data as TradeBar;
387  if (bigTradeBar != null)
388  {
389  return ToCsv(longTime, ToNonScaledCsv(bigTradeBar), bigTradeBar.Volume);
390  }
391  var bigOpenInterest = data as OpenInterest;
392  if (bigOpenInterest != null)
393  {
394  return ToCsv(longTime, bigOpenInterest.Value);
395  }
396  break;
397 
398  default:
399  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
400  }
401  break;
402 
403  case SecurityType.Future:
404  switch (resolution)
405  {
406  case Resolution.Tick:
407  var tick = (Tick)data;
408  if (tick.TickType == TickType.Trade)
409  {
410  return ToCsv(milliseconds,
411  tick.LastPrice, tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
412  }
413  if (tick.TickType == TickType.Quote)
414  {
415  return ToCsv(milliseconds,
416  tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.ExchangeCode, tick.Suspicious ? "1" : "0");
417  }
418  if (tick.TickType == TickType.OpenInterest)
419  {
420  return ToCsv(milliseconds, tick.Value);
421  }
422  break;
423 
424  case Resolution.Second:
425  case Resolution.Minute:
426  // option and future data can be quote or trade bars
427  var quoteBar = data as QuoteBar;
428  if (quoteBar != null)
429  {
430  return ToCsv(milliseconds,
431  ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
432  ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
433  }
434  var tradeBar = data as TradeBar;
435  if (tradeBar != null)
436  {
437  return ToCsv(milliseconds,
438  tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
439  }
440  var openInterest = data as OpenInterest;
441  if (openInterest != null)
442  {
443  return ToCsv(milliseconds, openInterest.Value);
444  }
445  break;
446 
447  case Resolution.Hour:
448  case Resolution.Daily:
449  // option and future data can be quote or trade bars
450  var bigQuoteBar = data as QuoteBar;
451  if (bigQuoteBar != null)
452  {
453  return ToCsv(longTime,
454  ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
455  ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
456  }
457  var bigTradeBar = data as TradeBar;
458  if (bigTradeBar != null)
459  {
460  return ToCsv(longTime, ToNonScaledCsv(bigTradeBar), bigTradeBar.Volume);
461  }
462  var bigOpenInterest = data as OpenInterest;
463  if (bigOpenInterest != null)
464  {
465  return ToCsv(longTime, bigOpenInterest.Value);
466  }
467  break;
468 
469  default:
470  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
471  }
472  break;
473 
474  default:
475  throw new ArgumentOutOfRangeException(nameof(securityType), securityType, null);
476  }
477 
478  throw new NotImplementedException(Invariant(
479  $"LeanData.GenerateLine has not yet been implemented for security type: {securityType} at resolution: {resolution}"
480  ));
481  }
482 
483  /// <summary>
484  /// Gets the data type required for the specified combination of resolution and tick type
485  /// </summary>
486  /// <param name="resolution">The resolution, if Tick, the Type returned is always Tick</param>
487  /// <param name="tickType">The <see cref="TickType"/> that primarily dictates the type returned</param>
488  /// <returns>The Type used to create a subscription</returns>
489  public static Type GetDataType(Resolution resolution, TickType tickType)
490  {
491  if (resolution == Resolution.Tick) return typeof(Tick);
492  if (tickType == TickType.OpenInterest) return typeof(OpenInterest);
493  if (tickType == TickType.Quote) return typeof(QuoteBar);
494  return typeof(TradeBar);
495  }
496 
497 
498  /// <summary>
499  /// Determines if the Type is a 'common' type used throughout lean
500  /// This method is helpful in creating <see cref="SubscriptionDataConfig"/>
501  /// </summary>
502  /// <param name="baseDataType">The Type to check</param>
503  /// <returns>A bool indicating whether the type is of type <see cref="TradeBar"/>
504  /// <see cref="QuoteBar"/> or <see cref="OpenInterest"/></returns>
505  public static bool IsCommonLeanDataType(Type baseDataType)
506  {
507  if (baseDataType == typeof(Tick) ||
508  baseDataType == typeof(TradeBar) ||
509  baseDataType == typeof(QuoteBar) ||
510  baseDataType == typeof(OpenInterest))
511  {
512  return true;
513  }
514 
515  return false;
516  }
517 
518  /// <summary>
519  /// Helper method to determine if a configuration set is valid
520  /// </summary>
521  public static bool IsValidConfiguration(SecurityType securityType, Resolution resolution, TickType tickType)
522  {
523  if (securityType == SecurityType.Equity && (resolution == Resolution.Daily || resolution == Resolution.Hour))
524  {
525  return tickType != TickType.Quote;
526  }
527  return true;
528  }
529 
530  /// <summary>
531  /// Generates the full zip file path rooted in the <paramref name="dataDirectory"/>
532  /// </summary>
533  public static string GenerateZipFilePath(string dataDirectory, Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
534  {
535  // we could call 'GenerateRelativeZipFilePath' but we don't to avoid an extra string & path combine we are doing to drop right away
536  return Path.Combine(dataDirectory, GenerateRelativeZipFileDirectory(symbol, resolution), GenerateZipFileName(symbol, date, resolution, tickType));
537  }
538 
539  /// <summary>
540  /// Generates the full zip file path rooted in the <paramref name="dataDirectory"/>
541  /// </summary>
542  public static string GenerateZipFilePath(string dataDirectory, string symbol, SecurityType securityType, string market, DateTime date, Resolution resolution)
543  {
544  return Path.Combine(dataDirectory, GenerateRelativeZipFilePath(symbol, securityType, market, date, resolution));
545  }
546 
547  /// <summary>
548  /// Generates the relative zip directory for the specified symbol/resolution
549  /// </summary>
550  public static string GenerateRelativeZipFileDirectory(Symbol symbol, Resolution resolution)
551  {
552  var isHourOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
553  var securityType = symbol.SecurityType.SecurityTypeToLower();
554 
555  var market = symbol.ID.Market.ToLowerInvariant();
556  var res = resolution.ResolutionToLower();
557  var directory = Path.Combine(securityType, market, res);
558  switch (symbol.ID.SecurityType)
559  {
560  case SecurityType.Base:
561  case SecurityType.Equity:
562  case SecurityType.Index:
563  case SecurityType.Forex:
564  case SecurityType.Cfd:
565  case SecurityType.Crypto:
566  return !isHourOrDaily ? Path.Combine(directory, symbol.Value.ToLowerInvariant()) : directory;
567 
568  case SecurityType.IndexOption:
569  // For index options, we use the canonical option ticker since it can differ from the underlying's ticker.
570  return !isHourOrDaily ? Path.Combine(directory, symbol.ID.Symbol.ToLowerInvariant()) : directory;
571 
572  case SecurityType.Option:
573  // options uses the underlying symbol for pathing.
574  return !isHourOrDaily ? Path.Combine(directory, symbol.Underlying.Value.ToLowerInvariant()) : directory;
575 
576  case SecurityType.FutureOption:
577  // For futures options, we use the canonical option ticker plus the underlying's expiry
578  // since it can differ from the underlying's ticker. We differ from normal futures
579  // because the option chain can be extraordinarily large compared to equity option chains.
580  var futureOptionPath = Path.Combine(symbol.ID.Symbol, symbol.Underlying.ID.Date.ToStringInvariant(DateFormat.EightCharacter))
581  .ToLowerInvariant();
582 
583  return Path.Combine(directory, futureOptionPath);
584 
585  case SecurityType.Future:
586  case SecurityType.CryptoFuture:
587  return !isHourOrDaily ? Path.Combine(directory, symbol.ID.Symbol.ToLowerInvariant()) : directory;
588 
589  case SecurityType.Commodity:
590  default:
591  throw new ArgumentOutOfRangeException();
592  }
593  }
594 
595  /// <summary>
596  /// Generates relative factor file paths for equities
597  /// </summary>
598  public static string GenerateRelativeFactorFilePath(Symbol symbol)
599  {
600  return Path.Combine(Globals.DataFolder,
601  "equity",
602  symbol.ID.Market,
603  "factor_files",
604  symbol.Value.ToLowerInvariant() + ".csv");
605  }
606 
607  /// <summary>
608  /// Generates the relative zip file path rooted in the /Data directory
609  /// </summary>
610  public static string GenerateRelativeZipFilePath(Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
611  {
612  return Path.Combine(GenerateRelativeZipFileDirectory(symbol, resolution), GenerateZipFileName(symbol, date, resolution, tickType));
613  }
614 
615  /// <summary>
616  /// Generates the relative zip file path rooted in the /Data directory
617  /// </summary>
618  public static string GenerateRelativeZipFilePath(string symbol, SecurityType securityType, string market, DateTime date, Resolution resolution)
619  {
620  var directory = Path.Combine(securityType.SecurityTypeToLower(), market.ToLowerInvariant(), resolution.ResolutionToLower());
621  if (resolution != Resolution.Daily && resolution != Resolution.Hour)
622  {
623  directory = Path.Combine(directory, symbol.ToLowerInvariant());
624  }
625 
626  return Path.Combine(directory, GenerateZipFileName(symbol, securityType, date, resolution));
627  }
628 
629  /// <summary>
630  /// Generates the relative directory to the universe files for the specified symbol
631  /// </summary>
632  public static string GenerateRelativeUniversesDirectory(Symbol symbol)
633  {
634  var path = Path.Combine(symbol.SecurityType.SecurityTypeToLower(), symbol.ID.Market, "universes");
635  switch (symbol.SecurityType)
636  {
637  case SecurityType.Option:
638  path = Path.Combine(path, symbol.Underlying.Value.ToLowerInvariant());
639  break;
640 
641  case SecurityType.IndexOption:
642  path = Path.Combine(path, symbol.ID.Symbol.ToLowerInvariant());
643  break;
644 
645  case SecurityType.FutureOption:
646  path = Path.Combine(path,
647  symbol.Underlying.Value.ToLowerInvariant(),
648  symbol.Underlying.ID.Date.ToStringInvariant(DateFormat.EightCharacter));
649  break;
650  }
651 
652  return path;
653  }
654 
655  /// <summary>
656  /// Generates the directory to the universe files for the specified symbol
657  /// </summary>
658  public static string GenerateUniversesDirectory(string dataDirectory, Symbol symbol)
659  {
660  return Path.Combine(dataDirectory, GenerateRelativeUniversesDirectory(symbol));
661  }
662 
663  /// <summary>
664  /// Generate's the zip entry name to hold the specified data.
665  /// </summary>
666  public static string GenerateZipEntryName(Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
667  {
668  var formattedDate = date.ToStringInvariant(DateFormat.EightCharacter);
669  var isHourOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
670 
671  switch (symbol.ID.SecurityType)
672  {
673  case SecurityType.Base:
674  case SecurityType.Equity:
675  case SecurityType.Index:
676  case SecurityType.Forex:
677  case SecurityType.Cfd:
678  case SecurityType.Crypto:
679  if (resolution == Resolution.Tick && symbol.SecurityType == SecurityType.Equity)
680  {
681  return Invariant($"{formattedDate}_{symbol.Value.ToLowerInvariant()}_{tickType}_{resolution}.csv");
682  }
683 
684  if (isHourOrDaily)
685  {
686  return $"{symbol.Value.ToLowerInvariant()}.csv";
687  }
688 
689  return Invariant($"{formattedDate}_{symbol.Value.ToLowerInvariant()}_{resolution.ResolutionToLower()}_{tickType.TickTypeToLower()}.csv");
690 
691  case SecurityType.Option:
692  var optionPath = symbol.Underlying.Value.ToLowerInvariant();
693 
694  if (isHourOrDaily)
695  {
696  return string.Join("_",
697  optionPath,
698  tickType.TickTypeToLower(),
699  symbol.ID.OptionStyle.OptionStyleToLower(),
700  symbol.ID.OptionRight.OptionRightToLower(),
701  Scale(symbol.ID.StrikePrice),
702  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
703  ) + ".csv";
704  }
705 
706  return string.Join("_",
707  formattedDate,
708  optionPath,
709  resolution.ResolutionToLower(),
710  tickType.TickTypeToLower(),
711  symbol.ID.OptionStyle.OptionStyleToLower(),
712  symbol.ID.OptionRight.OptionRightToLower(),
713  Scale(symbol.ID.StrikePrice),
714  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
715  ) + ".csv";
716 
717  case SecurityType.IndexOption:
718  case SecurityType.FutureOption:
719  // We want the future/index option ticker as the lookup name inside the ZIP file
720  var optionTickerBasedPath = symbol.ID.Symbol.ToLowerInvariant();
721 
722  if (isHourOrDaily)
723  {
724  return string.Join("_",
725  optionTickerBasedPath,
726  tickType.TickTypeToLower(),
727  symbol.ID.OptionStyle.OptionStyleToLower(),
728  symbol.ID.OptionRight.OptionRightToLower(),
729  Scale(symbol.ID.StrikePrice),
730  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
731  ) + ".csv";
732  }
733 
734  return string.Join("_",
735  formattedDate,
736  optionTickerBasedPath,
737  resolution.ResolutionToLower(),
738  tickType.TickTypeToLower(),
739  symbol.ID.OptionStyle.OptionStyleToLower(),
740  symbol.ID.OptionRight.OptionRightToLower(),
741  Scale(symbol.ID.StrikePrice),
742  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
743  ) + ".csv";
744 
745  case SecurityType.Future:
746  case SecurityType.CryptoFuture:
747  if (symbol.HasUnderlying)
748  {
749  symbol = symbol.Underlying;
750  }
751 
752  string expirationTag;
753  var expiryDate = symbol.ID.Date;
754  if (expiryDate != SecurityIdentifier.DefaultDate)
755  {
757  var contractYearMonth = expiryDate.AddMonths(monthsToAdd).ToStringInvariant(DateFormat.YearMonth);
758 
759  expirationTag = $"{contractYearMonth}_{expiryDate.ToStringInvariant(DateFormat.EightCharacter)}";
760  }
761  else
762  {
763  expirationTag = "perp";
764  }
765 
766  if (isHourOrDaily)
767  {
768  return string.Join("_",
769  symbol.ID.Symbol.ToLowerInvariant(),
770  tickType.TickTypeToLower(),
771  expirationTag
772  ) + ".csv";
773  }
774 
775  return string.Join("_",
776  formattedDate,
777  symbol.ID.Symbol.ToLowerInvariant(),
778  resolution.ResolutionToLower(),
779  tickType.TickTypeToLower(),
780  expirationTag
781  ) + ".csv";
782 
783  case SecurityType.Commodity:
784  default:
785  throw new ArgumentOutOfRangeException();
786  }
787  }
788 
789  /// <summary>
790  /// Generates the zip file name for the specified date of data.
791  /// </summary>
792  public static string GenerateZipFileName(Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
793  {
794  var tickTypeString = tickType.TickTypeToLower();
795  var formattedDate = date.ToStringInvariant(DateFormat.EightCharacter);
796  var isHourOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
797 
798  switch (symbol.ID.SecurityType)
799  {
800  case SecurityType.Base:
801  case SecurityType.Index:
802  case SecurityType.Equity:
803  case SecurityType.Forex:
804  case SecurityType.Cfd:
805  if (isHourOrDaily)
806  {
807  return $"{symbol.Value.ToLowerInvariant()}.zip";
808  }
809 
810  return $"{formattedDate}_{tickTypeString}.zip";
811  case SecurityType.Crypto:
812  if (isHourOrDaily)
813  {
814  return $"{symbol.Value.ToLowerInvariant()}_{tickTypeString}.zip";
815  }
816 
817  return $"{formattedDate}_{tickTypeString}.zip";
818  case SecurityType.Option:
819  if (isHourOrDaily)
820  {
821  // see TryParsePath: he knows tick type position is 3
822  var optionPath = symbol.Underlying.Value.ToLowerInvariant();
823  return $"{optionPath}_{date.Year}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
824  }
825 
826  return $"{formattedDate}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
827 
828  case SecurityType.IndexOption:
829  case SecurityType.FutureOption:
830  if (isHourOrDaily)
831  {
832  // see TryParsePath: he knows tick type position is 3
833  var optionTickerBasedPath = symbol.ID.Symbol.ToLowerInvariant();
834  return $"{optionTickerBasedPath}_{date.Year}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
835  }
836 
837  return $"{formattedDate}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
838 
839  case SecurityType.Future:
840  case SecurityType.CryptoFuture:
841  if (isHourOrDaily)
842  {
843  return $"{symbol.ID.Symbol.ToLowerInvariant()}_{tickTypeString}.zip";
844  }
845 
846  return $"{formattedDate}_{tickTypeString}.zip";
847 
848  case SecurityType.Commodity:
849  default:
850  throw new ArgumentOutOfRangeException();
851  }
852  }
853 
854  /// <summary>
855  /// Creates the zip file name for a QC zip data file
856  /// </summary>
857  public static string GenerateZipFileName(string symbol, SecurityType securityType, DateTime date, Resolution resolution, TickType? tickType = null)
858  {
859  if (resolution == Resolution.Hour || resolution == Resolution.Daily)
860  {
861  return $"{symbol.ToLowerInvariant()}.zip";
862  }
863 
864  var zipFileName = date.ToStringInvariant(DateFormat.EightCharacter);
865 
866  if (tickType == null)
867  {
868  if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd) {
869  tickType = TickType.Quote;
870  }
871  else
872  {
873  tickType = TickType.Trade;
874  }
875  }
876 
877  var suffix = Invariant($"_{tickType.Value.TickTypeToLower()}.zip");
878  return zipFileName + suffix;
879  }
880 
881  /// <summary>
882  /// Gets the tick type most commonly associated with the specified security type
883  /// </summary>
884  /// <param name="securityType">The security type</param>
885  /// <returns>The most common tick type for the specified security type</returns>
886  public static TickType GetCommonTickType(SecurityType securityType)
887  {
888  if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd || securityType == SecurityType.Crypto)
889  {
890  return TickType.Quote;
891  }
892  return TickType.Trade;
893  }
894 
895  /// <summary>
896  /// Creates a symbol from the specified zip entry name
897  /// </summary>
898  /// <param name="symbol">The root symbol of the output symbol</param>
899  /// <param name="resolution">The resolution of the data source producing the zip entry name</param>
900  /// <param name="zipEntryName">The zip entry name to be parsed</param>
901  /// <returns>A new symbol representing the zip entry name</returns>
902  public static Symbol ReadSymbolFromZipEntry(Symbol symbol, Resolution resolution, string zipEntryName)
903  {
904  var isHourlyOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
905  var parts = zipEntryName.Replace(".csv", string.Empty).Split('_');
906  switch (symbol.ID.SecurityType)
907  {
908  case SecurityType.Option:
909  case SecurityType.FutureOption:
910  case SecurityType.IndexOption:
911  if (isHourlyOrDaily)
912  {
913  var style = parts[2].ParseOptionStyle();
914  var right = parts[3].ParseOptionRight();
915  var strike = Parse.Decimal(parts[4]) / 10000m;
916  var expiry = Parse.DateTimeExact(parts[5], DateFormat.EightCharacter);
917  return Symbol.CreateOption(symbol.Underlying, symbol.ID.Symbol, symbol.ID.Market, style, right, strike, expiry);
918  }
919  else
920  {
921  var style = parts[4].ParseOptionStyle();
922  var right = parts[5].ParseOptionRight();
923  var strike = Parse.Decimal(parts[6]) / 10000m;
924  var expiry = DateTime.ParseExact(parts[7], DateFormat.EightCharacter, CultureInfo.InvariantCulture);
925  return Symbol.CreateOption(symbol.Underlying, symbol.ID.Symbol, symbol.ID.Market, style, right, strike, expiry);
926  }
927 
928  case SecurityType.Future:
929  if (isHourlyOrDaily)
930  {
931  var expiryYearMonth = Parse.DateTimeExact(parts[2], DateFormat.YearMonth);
932  var futureExpiryFunc = FuturesExpiryFunctions.FuturesExpiryFunction(symbol);
933  var futureExpiry = futureExpiryFunc(expiryYearMonth);
934  return Symbol.CreateFuture(parts[0], symbol.ID.Market, futureExpiry);
935  }
936  else
937  {
938  var expiryYearMonth = Parse.DateTimeExact(parts[4], DateFormat.YearMonth);
939  var futureExpiryFunc = FuturesExpiryFunctions.FuturesExpiryFunction(symbol);
940  var futureExpiry = futureExpiryFunc(expiryYearMonth);
941  return Symbol.CreateFuture(parts[1], symbol.ID.Market, futureExpiry);
942  }
943 
944  default:
945  throw new NotImplementedException(Invariant(
946  $"ReadSymbolFromZipEntry is not implemented for {symbol.ID.SecurityType} {symbol.ID.Market} {resolution}"
947  ));
948  }
949  }
950 
951  /// <summary>
952  /// Scale and convert the resulting number to deci-cents int.
953  /// </summary>
954  private static long Scale(decimal value)
955  {
956  return (long)(value * 10000);
957  }
958 
959  /// <summary>
960  /// Create a csv line from the specified arguments
961  /// </summary>
962  private static string ToCsv(params object[] args)
963  {
964  // use culture neutral formatting for decimals
965  for (var i = 0; i < args.Length; i++)
966  {
967  var value = args[i];
968  if (value is decimal)
969  {
970  args[i] = ((decimal)value).Normalize();
971  }
972  }
973 
974  var argsFormatted = args.Select(x => Convert.ToString(x, CultureInfo.InvariantCulture));
975  return string.Join(",", argsFormatted);
976  }
977 
978  /// <summary>
979  /// Creates a scaled csv line for the bar, if null fills in empty strings
980  /// </summary>
981  private static string ToScaledCsv(IBar bar)
982  {
983  if (bar == null)
984  {
985  return ToCsv(string.Empty, string.Empty, string.Empty, string.Empty);
986  }
987 
988  return ToCsv(Scale(bar.Open), Scale(bar.High), Scale(bar.Low), Scale(bar.Close));
989  }
990 
991 
992  /// <summary>
993  /// Creates a non scaled csv line for the bar, if null fills in empty strings
994  /// </summary>
995  private static string ToNonScaledCsv(IBar bar)
996  {
997  if (bar == null)
998  {
999  return ToCsv(string.Empty, string.Empty, string.Empty, string.Empty);
1000  }
1001 
1002  return ToCsv(bar.Open, bar.High, bar.Low, bar.Close);
1003  }
1004 
1005  /// <summary>
1006  /// Get the <see cref="TickType"/> for common Lean data types.
1007  /// If not a Lean common data type, return a TickType of Trade.
1008  /// </summary>
1009  /// <param name="type">A Type used to determine the TickType</param>
1010  /// <param name="securityType">The SecurityType used to determine the TickType</param>
1011  /// <returns>A TickType corresponding to the type</returns>
1012  public static TickType GetCommonTickTypeForCommonDataTypes(Type type, SecurityType securityType)
1013  {
1014  if (type == typeof(TradeBar))
1015  {
1016  return TickType.Trade;
1017  }
1018  if (type == typeof(QuoteBar))
1019  {
1020  return TickType.Quote;
1021  }
1022  if (type == typeof(OpenInterest))
1023  {
1024  return TickType.OpenInterest;
1025  }
1026  if (type == typeof(ZipEntryName))
1027  {
1028  return TickType.Quote;
1029  }
1030  if (type == typeof(Tick))
1031  {
1032  return GetCommonTickType(securityType);
1033  }
1034 
1035  return TickType.Trade;
1036  }
1037 
1038  /// <summary>
1039  /// Matches a data path security type with the <see cref="SecurityType"/>
1040  /// </summary>
1041  /// <remarks>This includes 'alternative'</remarks>
1042  /// <param name="securityType">The data path security type</param>
1043  /// <returns>The matching security type for the given data path</returns>
1044  public static SecurityType ParseDataSecurityType(string securityType)
1045  {
1046  if (securityType.Equals("alternative", StringComparison.InvariantCultureIgnoreCase))
1047  {
1048  return SecurityType.Base;
1049  }
1050  return (SecurityType)Enum.Parse(typeof(SecurityType), securityType, true);
1051  }
1052 
1053  /// <summary>
1054  /// Parses file name into a <see cref="Security"/> and DateTime
1055  /// </summary>
1056  /// <param name="fileName">File name to be parsed</param>
1057  /// <param name="securityType">The securityType as parsed from the fileName</param>
1058  /// <param name="market">The market as parsed from the fileName</param>
1059  public static bool TryParseSecurityType(string fileName, out SecurityType securityType, out string market)
1060  {
1061  securityType = SecurityType.Base;
1062  market = string.Empty;
1063 
1064  try
1065  {
1066  var info = SplitDataPath(fileName);
1067 
1068  // find the securityType and parse it
1069  var typeString = info.Find(x => SecurityTypeAsDataPath.Contains(x.ToLowerInvariant()));
1070  securityType = ParseDataSecurityType(typeString);
1071 
1072  var existingMarkets = Market.SupportedMarkets();
1073  var foundMarket = info.Find(x => existingMarkets.Contains(x.ToLowerInvariant()));
1074  if (foundMarket != null)
1075  {
1076  market = foundMarket;
1077  }
1078  }
1079  catch (Exception e)
1080  {
1081  Log.Error($"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {e.GetBaseException()}");
1082  return false;
1083  }
1084 
1085  return true;
1086  }
1087 
1088  /// <summary>
1089  /// Parses file name into a <see cref="Security"/> and DateTime
1090  /// </summary>
1091  /// <param name="filePath">File path to be parsed</param>
1092  /// <param name="symbol">The symbol as parsed from the fileName</param>
1093  /// <param name="date">Date of data in the file path. Only returned if the resolution is lower than Hourly</param>
1094  /// <param name="resolution">The resolution of the symbol as parsed from the filePath</param>
1095  /// <param name="tickType">The tick type</param>
1096  /// <param name="dataType">The data type</param>
1097  public static bool TryParsePath(string filePath, out Symbol symbol, out DateTime date,
1098  out Resolution resolution, out TickType tickType, out Type dataType)
1099  {
1100  symbol = default;
1101  tickType = default;
1102  dataType = default;
1103  date = default;
1104  resolution = default;
1105 
1106  try
1107  {
1108  if (!TryParsePath(filePath, out symbol, out date, out resolution))
1109  {
1110  return false;
1111  }
1112 
1113  tickType = GetCommonTickType(symbol.SecurityType);
1114  var fileName = Path.GetFileNameWithoutExtension(filePath);
1115  if (fileName.Contains('_', StringComparison.InvariantCulture))
1116  {
1117  // example: 20140606_openinterest_american.zip
1118  var tickTypePosition = 1;
1119  if (resolution >= Resolution.Hour && symbol.SecurityType.IsOption())
1120  {
1121  // daily and hourly have the year too, example: aapl_2014_openinterest_american.zip
1122  // see GenerateZipFileName he's creating these paths
1123  tickTypePosition = 2;
1124  }
1125  tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[tickTypePosition], true);
1126  }
1127 
1128  dataType = GetDataType(resolution, tickType);
1129  return true;
1130  }
1131  catch (Exception ex)
1132  {
1133  Log.Debug($"LeanData.TryParsePath(): Error encountered while parsing the path {filePath}. Error: {ex.GetBaseException()}");
1134  }
1135  return false;
1136  }
1137 
1138  /// <summary>
1139  /// Parses file name into a <see cref="Security"/> and DateTime
1140  /// </summary>
1141  /// <param name="fileName">File name to be parsed</param>
1142  /// <param name="symbol">The symbol as parsed from the fileName</param>
1143  /// <param name="date">Date of data in the file path. Only returned if the resolution is lower than Hourly</param>
1144  /// <param name="resolution">The resolution of the symbol as parsed from the filePath</param>
1145  public static bool TryParsePath(string fileName, out Symbol symbol, out DateTime date, out Resolution resolution)
1146  {
1147  symbol = null;
1148  resolution = Resolution.Daily;
1149  date = default(DateTime);
1150 
1151  try
1152  {
1153  var info = SplitDataPath(fileName);
1154 
1155  // find where the useful part of the path starts - i.e. the securityType
1156  var startIndex = info.FindIndex(x => SecurityTypeAsDataPath.Contains(x.ToLowerInvariant()));
1157 
1158  if (startIndex == -1)
1159  {
1160  if (Log.DebuggingEnabled)
1161  {
1162  Log.Debug($"LeanData.TryParsePath(): Failed to parse '{fileName}' unexpected SecurityType");
1163  }
1164  // SPDB & MHDB folders
1165  return false;
1166  }
1167  var securityType = ParseDataSecurityType(info[startIndex]);
1168 
1169  var market = Market.USA;
1170  string ticker;
1171  var isUniverses = false;
1172  if (!Enum.TryParse(info[startIndex + 2], true, out resolution))
1173  {
1174  resolution = Resolution.Daily;
1175  isUniverses = info[startIndex + 2].Equals("universes", StringComparison.InvariantCultureIgnoreCase);
1176  if (securityType != SecurityType.Base)
1177  {
1178  if (!isUniverses)
1179  {
1180  if (Log.DebuggingEnabled)
1181  {
1182  Log.Debug($"LeanData.TryParsePath(): Failed to parse '{fileName}' unexpected Resolution");
1183  }
1184  // only acept a failure to parse resolution if we are facing a universes path
1185  return false;
1186  }
1187  securityType = SecurityType.Base;
1188  }
1189  }
1190 
1191  if (securityType == SecurityType.Base)
1192  {
1193  // the last part of the path is the file name
1194  var fileNameNoPath = info[info.Count - 1].Split('_').First();
1195 
1196  if (!DateTime.TryParseExact(fileNameNoPath,
1198  DateTimeFormatInfo.InvariantInfo,
1199  DateTimeStyles.None,
1200  out date))
1201  {
1202  // if parsing the date failed we assume filename is ticker
1203  ticker = fileNameNoPath;
1204  }
1205  else
1206  {
1207  // ticker must be the previous part of the path
1208  ticker = info[info.Count - 2];
1209  }
1210  }
1211  else
1212  {
1213  // Gather components used to create the security
1214  market = info[startIndex + 1];
1215  var components = info[startIndex + 3].Split('_');
1216 
1217  // Remove the ticktype from the ticker (Only exists in Crypto and Future data but causes no issues)
1218  ticker = components[0];
1219 
1220  if (resolution < Resolution.Hour)
1221  {
1222  // Future options are special and have the following format Market/Resolution/Ticker/FutureExpiry/Date
1223  var dateIndex = securityType == SecurityType.FutureOption ? startIndex + 5 : startIndex + 4;
1224  date = Parse.DateTimeExact(info[dateIndex].Substring(0, 8), DateFormat.EightCharacter);
1225  }
1226  // If resolution is Daily or Hour for options and index options, we can only get the year from the path
1227  else if (securityType == SecurityType.Option || securityType == SecurityType.IndexOption)
1228  {
1229  var year = int.Parse(components[1], CultureInfo.InvariantCulture);
1230  date = new DateTime(year, 01, 01);
1231  }
1232  }
1233 
1234  // Future Options cannot use Symbol.Create
1235  if (securityType == SecurityType.FutureOption)
1236  {
1237  // Future options have underlying FutureExpiry date as the parent dir for the zips, we need this for our underlying
1238  var underlyingFutureExpiryDate = Parse.DateTimeExact(info[startIndex + 4].Substring(0, 8), DateFormat.EightCharacter);
1239 
1240  var underlyingTicker = OptionSymbol.MapToUnderlying(ticker, securityType);
1241  // Create our underlying future and then the Canonical option for this future
1242  var underlyingFuture = Symbol.CreateFuture(underlyingTicker, market, underlyingFutureExpiryDate);
1243  symbol = Symbol.CreateCanonicalOption(underlyingFuture);
1244  }
1245  else if (securityType == SecurityType.IndexOption)
1246  {
1247  var underlyingTicker = OptionSymbol.MapToUnderlying(ticker, securityType);
1248  // Create our underlying index and then the Canonical option
1249  var underlyingIndex = Symbol.Create(underlyingTicker, SecurityType.Index, market);
1250  symbol = Symbol.CreateCanonicalOption(underlyingIndex, ticker, market, null);
1251  }
1252  else
1253  {
1254  Type dataType = null;
1255  if (isUniverses && info[startIndex + 3].Equals("etf", StringComparison.InvariantCultureIgnoreCase))
1256  {
1257  dataType = typeof(ETFConstituentUniverse);
1258  }
1259  symbol = CreateSymbol(ticker, securityType, market, dataType, date);
1260  }
1261 
1262  }
1263  catch (Exception ex)
1264  {
1265  Log.Debug($"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {ex.GetBaseException()}");
1266  return false;
1267  }
1268 
1269  return true;
1270  }
1271 
1272  /// <summary>
1273  /// Creates a new Symbol based on parsed data path information.
1274  /// </summary>
1275  /// <param name="ticker">The parsed ticker symbol.</param>
1276  /// <param name="securityType">The parsed type of security.</param>
1277  /// <param name="market">The parsed market or exchange.</param>
1278  /// <param name="dataType">Optional type used for generating the base data SID (applicable only for SecurityType.Base).</param>
1279  /// <param name="mappingResolveDate">The date used in path parsing to create the correct symbol.</param>
1280  /// <returns>A unique security identifier.</returns>
1281  /// <example>
1282  /// <code>
1283  /// path: equity/usa/minute/spwr/20071223_trade.zip
1284  /// ticker: spwr
1285  /// securityType: equity
1286  /// market: usa
1287  /// mappingResolveDate: 2007/12/23
1288  /// </code>
1289  /// </example>
1290  private static Symbol CreateSymbol(string ticker, SecurityType securityType, string market, Type dataType, DateTime mappingResolveDate = default)
1291  {
1292  if (mappingResolveDate != default && (securityType == SecurityType.Equity || securityType == SecurityType.Option))
1293  {
1294  var symbol = new Symbol(SecurityIdentifier.GenerateEquity(ticker, market, mappingResolveDate: mappingResolveDate), ticker);
1295  return securityType == SecurityType.Option ? Symbol.CreateCanonicalOption(symbol) : symbol;
1296  }
1297  else
1298  {
1299  return Symbol.Create(ticker, securityType, market, baseDataType: dataType);
1300  }
1301  }
1302 
1303  private static List<string> SplitDataPath(string fileName)
1304  {
1305  var pathSeparators = new[] { '/', '\\' };
1306 
1307  // Removes file extension
1308  fileName = fileName.Replace(fileName.GetExtension(), string.Empty);
1309 
1310  // remove any relative file path
1311  while (fileName.First() == '.' || pathSeparators.Any(x => x == fileName.First()))
1312  {
1313  fileName = fileName.Remove(0, 1);
1314  }
1315 
1316  // split path into components
1317  return fileName.Split(pathSeparators, StringSplitOptions.RemoveEmptyEntries).ToList();
1318  }
1319 
1320  /// <summary>
1321  /// Aggregates a list of second/minute bars at the requested resolution
1322  /// </summary>
1323  /// <param name="bars">List of <see cref="TradeBar"/>s</param>
1324  /// <param name="symbol">Symbol of all tradeBars</param>
1325  /// <param name="resolution">Desired resolution for new <see cref="TradeBar"/>s</param>
1326  /// <returns>List of aggregated <see cref="TradeBar"/>s</returns>
1327  public static IEnumerable<TradeBar> AggregateTradeBars(IEnumerable<TradeBar> bars, Symbol symbol, TimeSpan resolution)
1328  {
1329  return Aggregate(new TradeBarConsolidator(resolution), bars, symbol);
1330  }
1331 
1332  /// <summary>
1333  /// Aggregates a list of second/minute bars at the requested resolution
1334  /// </summary>
1335  /// <param name="bars">List of <see cref="QuoteBar"/>s</param>
1336  /// <param name="symbol">Symbol of all QuoteBars</param>
1337  /// <param name="resolution">Desired resolution for new <see cref="QuoteBar"/>s</param>
1338  /// <returns>List of aggregated <see cref="QuoteBar"/>s</returns>
1339  public static IEnumerable<QuoteBar> AggregateQuoteBars(IEnumerable<QuoteBar> bars, Symbol symbol, TimeSpan resolution)
1340  {
1341  return Aggregate(new QuoteBarConsolidator(resolution), bars, symbol);
1342  }
1343 
1344  /// <summary>
1345  /// Aggregates a list of ticks at the requested resolution
1346  /// </summary>
1347  /// <param name="ticks">List of quote ticks</param>
1348  /// <param name="symbol">Symbol of all ticks</param>
1349  /// <param name="resolution">Desired resolution for new <see cref="QuoteBar"/>s</param>
1350  /// <returns>List of aggregated <see cref="QuoteBar"/>s</returns>
1351  public static IEnumerable<QuoteBar> AggregateTicks(IEnumerable<Tick> ticks, Symbol symbol, TimeSpan resolution)
1352  {
1353  return Aggregate(new TickQuoteBarConsolidator(resolution), ticks, symbol);
1354  }
1355 
1356  /// <summary>
1357  /// Aggregates a list of ticks at the requested resolution
1358  /// </summary>
1359  /// <param name="ticks">List of trade ticks</param>
1360  /// <param name="symbol">Symbol of all ticks</param>
1361  /// <param name="resolution">Desired resolution for new <see cref="TradeBar"/>s</param>
1362  /// <returns>List of aggregated <see cref="TradeBar"/>s</returns>
1363  public static IEnumerable<TradeBar> AggregateTicksToTradeBars(IEnumerable<Tick> ticks, Symbol symbol, TimeSpan resolution)
1364  {
1365  return Aggregate(new TickConsolidator(resolution), ticks, symbol);
1366  }
1367 
1368  /// <summary>
1369  /// Helper method to return the start time and period of a bar the given point time should be part of
1370  /// </summary>
1371  /// <param name="exchangeTimeZoneDate">The point in time we want to get the bar information about</param>
1372  /// <param name="exchange">The associated security exchange</param>
1373  /// <param name="extendedMarketHours">True if extended market hours should be taken into consideration</param>
1374  /// <returns>The calendar information that holds a start time and a period</returns>
1375  public static CalendarInfo GetDailyCalendar(DateTime exchangeTimeZoneDate, SecurityExchange exchange, bool extendedMarketHours)
1376  {
1377  return GetDailyCalendar(exchangeTimeZoneDate, exchange.Hours, extendedMarketHours);
1378  }
1379 
1380  /// <summary>
1381  /// Helper method to return the start time and period of a bar the given point time should be part of
1382  /// </summary>
1383  /// <param name="exchangeTimeZoneDate">The point in time we want to get the bar information about</param>
1384  /// <param name="exchangeHours">The associated exchange hours</param>
1385  /// <param name="extendedMarketHours">True if extended market hours should be taken into consideration</param>
1386  /// <returns>The calendar information that holds a start time and a period</returns>
1387  public static CalendarInfo GetDailyCalendar(DateTime exchangeTimeZoneDate, SecurityExchangeHours exchangeHours, bool extendedMarketHours)
1388  {
1389  var startTime = exchangeHours.GetPreviousMarketOpen(exchangeTimeZoneDate, extendedMarketHours);
1390  var endTime = exchangeHours.GetNextMarketClose(startTime, extendedMarketHours);
1391 
1392  // Let's not consider regular market gaps like when market closes at 16:15 and opens again at 16:30
1393  while (true)
1394  {
1395  var potentialEnd = exchangeHours.GetNextMarketClose(endTime, extendedMarketHours);
1396  if (potentialEnd.Date != endTime.Date)
1397  {
1398  break;
1399  }
1400  endTime = potentialEnd;
1401  }
1402 
1403  var period = endTime - startTime;
1404  return new CalendarInfo(startTime, period);
1405  }
1406 
1407  /// <summary>
1408  /// Helper method to get the next daily end time, taking into account strict end times if appropriate
1409  /// </summary>
1410  public static DateTime GetNextDailyEndTime(Symbol symbol, DateTime exchangeTimeZoneDate, SecurityExchangeHours exchangeHours)
1411  {
1412  var nextMidnight = exchangeTimeZoneDate.Date.AddDays(1);
1413  if (!UseStrictEndTime(true, symbol, Time.OneDay, exchangeHours))
1414  {
1415  return nextMidnight;
1416  }
1417 
1418  var nextMarketClose = exchangeHours.GetNextMarketClose(exchangeTimeZoneDate, extendedMarketHours: false);
1419  if (nextMarketClose > nextMidnight)
1420  {
1421  // if exchangeTimeZoneDate is after the previous close, the next close might be tomorrow
1422  if (!exchangeHours.IsOpen(exchangeTimeZoneDate, extendedMarketHours: false))
1423  {
1424  return nextMarketClose;
1425  }
1426  return nextMidnight;
1427  }
1428  return nextMarketClose;
1429  }
1430 
1431  /// <summary>
1432  /// Helper method that defines the types of options that should use scale factor
1433  /// </summary>
1434  [MethodImpl(MethodImplOptions.AggressiveInlining)]
1435  public static bool OptionUseScaleFactor(Symbol symbol)
1436  {
1437  return symbol.SecurityType == SecurityType.Option || symbol.SecurityType == SecurityType.IndexOption;
1438  }
1439 
1440  /// <summary>
1441  /// Helper method to determine if we should use strict end time
1442  /// </summary>
1443  /// <param name="symbol">The associated symbol</param>
1444  /// <param name="increment">The datas time increment</param>
1445  [MethodImpl(MethodImplOptions.AggressiveInlining)]
1446  public static bool UseStrictEndTime(bool dailyStrictEndTimeEnabled, Symbol symbol, TimeSpan increment, SecurityExchangeHours exchangeHours)
1447  {
1448  if (exchangeHours.IsMarketAlwaysOpen
1449  || increment <= Time.OneHour
1450  || symbol.SecurityType == SecurityType.Cfd && symbol.ID.Market == Market.Oanda
1451  || symbol.SecurityType == SecurityType.Forex
1452  || symbol.SecurityType == SecurityType.Base)
1453  {
1454  return false;
1455  }
1456  return dailyStrictEndTimeEnabled;
1457  }
1458 
1459  /// <summary>
1460  /// Helper method to determine if we should use strict end time
1461  /// </summary>
1462  public static bool UseDailyStrictEndTimes(IAlgorithmSettings settings, BaseDataRequest request, Symbol symbol, TimeSpan increment,
1463  SecurityExchangeHours exchangeHours = null)
1464  {
1465  return UseDailyStrictEndTimes(settings, request.DataType, symbol, increment, exchangeHours ?? request.ExchangeHours);
1466  }
1467 
1468  /// <summary>
1469  /// Helper method to determine if we should use strict end time
1470  /// </summary>
1471  public static bool UseDailyStrictEndTimes(IAlgorithmSettings settings, Type dataType, Symbol symbol, TimeSpan increment, SecurityExchangeHours exchangeHours)
1472  {
1473  return UseDailyStrictEndTimes(settings.DailyPreciseEndTime, dataType, symbol, increment, exchangeHours);
1474  }
1475 
1476  /// <summary>
1477  /// Helper method to determine if we should use strict end time
1478  /// </summary>
1479  public static bool UseDailyStrictEndTimes(bool dailyStrictEndTimeEnabled, Type dataType, Symbol symbol, TimeSpan increment, SecurityExchangeHours exchangeHours)
1480  {
1481  return UseDailyStrictEndTimes(dataType) && UseStrictEndTime(dailyStrictEndTimeEnabled, symbol, increment, exchangeHours);
1482  }
1483 
1484  /// <summary>
1485  /// True if this data type should use strict daily end times
1486  /// </summary>
1487  public static bool UseDailyStrictEndTimes(Type dataType)
1488  {
1489  return dataType != null && _strictDailyEndTimesDataTypes.Contains(dataType);
1490  }
1491 
1492  /// <summary>
1493  /// Helper method that if appropiate, will set the Time and EndTime of the given data point to it's daily strict times
1494  /// </summary>
1495  /// <param name="baseData">The target data point</param>
1496  /// <param name="exchange">The associated exchange hours</param>
1497  /// <remarks>This method is used to set daily times on pre existing data, assuming it does not cover extended market hours</remarks>
1498  [MethodImpl(MethodImplOptions.AggressiveInlining)]
1499  public static bool SetStrictEndTimes(IBaseData baseData, SecurityExchangeHours exchange)
1500  {
1501  if (baseData == null)
1502  {
1503  return false;
1504  }
1505 
1506  var dataType = baseData.GetType();
1507  if (!UseDailyStrictEndTimes(dataType))
1508  {
1509  return false;
1510  }
1511 
1512  var isZipEntryName = dataType == typeof(ZipEntryName);
1513  if (isZipEntryName && baseData.Time.Hour == 0)
1514  {
1515  // zip entry names are emitted point in time for a date, see BaseDataSubscriptionEnumeratorFactory. When setting the strict end times
1516  // we will move it to the previous day daily times, because daily market data on disk end time is midnight next day, so here we add 1 day
1517  baseData.Time += Time.OneDay;
1518  baseData.EndTime += Time.OneDay;
1519  }
1520 
1521  var dailyCalendar = GetDailyCalendar(baseData.EndTime, exchange, extendedMarketHours: false);
1522  if (!isZipEntryName && dailyCalendar.End < baseData.Time)
1523  {
1524  // this data point we were given is probably from extended market hours which we don't support for daily backtesting data
1525  return false;
1526  }
1527  baseData.Time = dailyCalendar.Start;
1528  baseData.EndTime = dailyCalendar.End;
1529  return true;
1530  }
1531 
1532  /// <summary>
1533  /// Helper to separate filename and entry from a given key for DataProviders
1534  /// </summary>
1535  /// <param name="key">The key to parse</param>
1536  /// <param name="fileName">File name extracted</param>
1537  /// <param name="entryName">Entry name extracted</param>
1538  public static void ParseKey(string key, out string fileName, out string entryName)
1539  {
1540  // Default scenario, no entryName included in key
1541  entryName = null; // default to all entries
1542  fileName = key;
1543 
1544  if (key == null)
1545  {
1546  return;
1547  }
1548 
1549  // Try extracting an entry name; Anything after a # sign
1550  var hashIndex = key.LastIndexOf("#", StringComparison.Ordinal);
1551  if (hashIndex != -1)
1552  {
1553  entryName = key.Substring(hashIndex + 1);
1554  fileName = key.Substring(0, hashIndex);
1555  }
1556  }
1557 
1558  /// <summary>
1559  /// Helper method to aggregate ticks or bars into lower frequency resolutions
1560  /// </summary>
1561  /// <typeparam name="T">Output type</typeparam>
1562  /// <typeparam name="K">Input type</typeparam>
1563  /// <param name="consolidator">The consolidator to use</param>
1564  /// <param name="dataPoints">The data point source</param>
1565  /// <param name="symbol">The symbol to output</param>
1566  private static IEnumerable<T> Aggregate<T, K>(PeriodCountConsolidatorBase<K, T> consolidator, IEnumerable<K> dataPoints, Symbol symbol)
1567  where T : BaseData
1568  where K : BaseData
1569  {
1570  IBaseData lastAggregated = null;
1571  var getConsolidatedBar = () =>
1572  {
1573  if (lastAggregated != consolidator.Consolidated && consolidator.Consolidated != null)
1574  {
1575  // if there's a new aggregated bar we set the symbol & return it
1576  lastAggregated = consolidator.Consolidated;
1577  lastAggregated.Symbol = symbol;
1578  return lastAggregated;
1579  }
1580  return null;
1581  };
1582 
1583  foreach (var dataPoint in dataPoints)
1584  {
1585  consolidator.Update(dataPoint);
1586  var consolidated = getConsolidatedBar();
1587  if (consolidated != null)
1588  {
1589  yield return (T)consolidated;
1590  }
1591  }
1592 
1593  // flush any partial bar
1594  consolidator.Scan(Time.EndOfTime);
1595  var lastConsolidated = getConsolidatedBar();
1596  if (lastConsolidated != null)
1597  {
1598  yield return (T)lastConsolidated;
1599  }
1600 
1601  // cleanup
1602  consolidator.DisposeSafely();
1603  }
1604  }
1605 }