Lean  $LEAN_TAG$
PandasData.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using Python.Runtime;
17 using QuantConnect.Data;
20 using QuantConnect.Util;
21 using System;
22 using System.Collections;
23 using System.Collections.Concurrent;
24 using System.Collections.Generic;
25 using System.Globalization;
26 using System.Linq;
27 using System.Reflection;
28 
29 namespace QuantConnect.Python
30 {
31  /// <summary>
32  /// Organizes a list of data to create pandas.DataFrames
33  /// </summary>
34  public class PandasData
35  {
36  private const string Open = "open";
37  private const string High = "high";
38  private const string Low = "low";
39  private const string Close = "close";
40  private const string Volume = "volume";
41 
42  private const string AskOpen = "askopen";
43  private const string AskHigh = "askhigh";
44  private const string AskLow = "asklow";
45  private const string AskClose = "askclose";
46  private const string AskPrice = "askprice";
47  private const string AskSize = "asksize";
48 
49  private const string BidOpen = "bidopen";
50  private const string BidHigh = "bidhigh";
51  private const string BidLow = "bidlow";
52  private const string BidClose = "bidclose";
53  private const string BidPrice = "bidprice";
54  private const string BidSize = "bidsize";
55 
56  private const string LastPrice = "lastprice";
57  private const string Quantity = "quantity";
58  private const string Exchange = "exchange";
59  private const string Suspicious = "suspicious";
60  private const string OpenInterest = "openinterest";
61 
62  #region OptionContract Members Handling
63 
64  // TODO: In the future, excluding, adding, renaming and unwrapping members (like the Greeks case)
65  // should be handled generically: we could define attributes so that class members can be marked as
66  // excluded, or to be renamed and/ or unwrapped (much like how Json attributes work)
67 
68  private static readonly string[] _optionContractExcludedMembers = new[]
69  {
70  nameof(OptionContract.ID),
71  };
72 
73  private static readonly string[] _greeksMemberNames = new[]
74  {
75  nameof(Greeks.Delta).ToLowerInvariant(),
76  nameof(Greeks.Gamma).ToLowerInvariant(),
77  nameof(Greeks.Vega).ToLowerInvariant(),
78  nameof(Greeks.Theta).ToLowerInvariant(),
79  nameof(Greeks.Rho).ToLowerInvariant(),
80  };
81 
82  private static readonly MemberInfo[] _greeksMembers = typeof(Greeks)
83  .GetMembers(BindingFlags.Instance | BindingFlags.Public)
84  .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) &&
85  _greeksMemberNames.Contains(x.Name.ToLowerInvariant()))
86  .ToArray();
87 
88  #endregion
89 
90  // we keep these so we don't need to ask for them each time
91  private static PyString _empty;
92  private static PyObject _pandas;
93  private static PyObject _pandasColumn;
94  private static PyObject _seriesFactory;
95  private static PyObject _dataFrameFactory;
96  private static PyObject _multiIndexFactory;
97  private static PyObject _multiIndex;
98  private static PyObject _indexFactory;
99 
100  private static PyList _defaultNames;
101  private static PyList _level1Names;
102  private static PyList _level2Names;
103  private static PyList _level3Names;
104 
105  private readonly static HashSet<string> _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant());
106  private readonly static ConcurrentDictionary<Type, IEnumerable<DataTypeMember>> _membersByType = new();
107  private readonly static IReadOnlyList<string> _standardColumns = new string[]
108  {
109  Open, High, Low, Close, LastPrice, Volume,
110  AskOpen, AskHigh, AskLow, AskClose, AskPrice, AskSize, Quantity, Suspicious,
111  BidOpen, BidHigh, BidLow, BidClose, BidPrice, BidSize, Exchange, OpenInterest
112  };
113 
114  private readonly Symbol _symbol;
115  private readonly bool _isFundamentalType;
116  private readonly bool _isBaseData;
117  private readonly Dictionary<string, Serie> _series;
118 
119  private readonly IEnumerable<DataTypeMember> _members = Enumerable.Empty<DataTypeMember>();
120 
121  /// <summary>
122  /// Gets true if this is a custom data request, false for normal QC data
123  /// </summary>
124  public bool IsCustomData { get; }
125 
126  /// <summary>
127  /// Implied levels of a multi index pandas.Series (depends on the security type)
128  /// </summary>
129  public int Levels { get; } = 2;
130 
131  /// <summary>
132  /// Initializes the static members of the <see cref="PandasData"/> class
133  /// </summary>
134  static PandasData()
135  {
136  using (Py.GIL())
137  {
138  // Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs
139  _pandas = Py.Import("PandasMapper");
140  _pandasColumn = _pandas.GetAttr("PandasColumn");
141  _seriesFactory = _pandas.GetAttr("Series");
142  _dataFrameFactory = _pandas.GetAttr("DataFrame");
143  _multiIndex = _pandas.GetAttr("MultiIndex");
144  _multiIndexFactory = _multiIndex.GetAttr("from_tuples");
145  _indexFactory = _pandas.GetAttr("Index");
146  _empty = new PyString(string.Empty);
147 
148  var time = new PyString("time");
149  var symbol = new PyString("symbol");
150  var expiry = new PyString("expiry");
151  _defaultNames = new PyList(new PyObject[] { expiry, new PyString("strike"), new PyString("type"), symbol, time });
152  _level1Names = new PyList(new PyObject[] { symbol });
153  _level2Names = new PyList(new PyObject[] { symbol, time });
154  _level3Names = new PyList(new PyObject[] { expiry, symbol, time });
155  }
156  }
157 
158  /// <summary>
159  /// Initializes an instance of <see cref="PandasData"/>
160  /// </summary>
161  public PandasData(object data)
162  {
163  var baseData = data as IBaseData;
164 
165  // in the case we get a list/collection of data we take the first data point to determine the type
166  // but it's also possible to get a data which supports enumerating we don't care about those cases
167  if (baseData == null && data is IEnumerable enumerable)
168  {
169  foreach (var item in enumerable)
170  {
171  data = item;
172  baseData = data as IBaseData;
173  break;
174  }
175  }
176 
177  var type = data.GetType();
178  _isFundamentalType = type == typeof(Fundamental);
179  _isBaseData = baseData != null;
180  _symbol = _isBaseData ? baseData.Symbol : ((ISymbolProvider)data).Symbol;
181  IsCustomData = Extensions.IsCustomDataType(_symbol, type);
182 
183  if (baseData == null)
184  {
185  Levels = 1;
186  }
187  else if (_symbol.SecurityType == SecurityType.Future)
188  {
189  Levels = 3;
190  }
191  else if (_symbol.SecurityType.IsOption())
192  {
193  Levels = 5;
194  }
195 
196  IEnumerable<string> columns = _standardColumns;
197 
198  if (IsCustomData || !_isBaseData || baseData.DataType == MarketDataType.Auxiliary)
199  {
200  var keys = (data as DynamicData)?.GetStorageDictionary()
201  // if this is a PythonData instance we add in '__typename' which we don't want into the data frame
202  .Where(x => !x.Key.StartsWith("__", StringComparison.InvariantCulture)).ToHashSet(x => x.Key);
203 
204  // C# types that are not DynamicData type
205  if (keys == null)
206  {
207  if (_membersByType.TryGetValue(type, out _members))
208  {
209  keys = _members.SelectMany(x => x.GetMemberNames()).ToHashSet();
210  }
211  else
212  {
213  var members = type
214  .GetMembers(BindingFlags.Instance | BindingFlags.Public)
215  .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property);
216 
217  // TODO: Avoid hard-coded especial cases by using something like attributes to change
218  // pandas conversion behavior
219  if (type.IsAssignableTo(typeof(OptionContract)))
220  {
221  members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name));
222  }
223 
224  var dataTypeMembers = members.Select(x =>
225  {
226  if (!DataTypeMember.GetMemberType(x).IsAssignableTo(typeof(Greeks)))
227  {
228  return new DataTypeMember(x);
229  }
230 
231  return new DataTypeMember(x, _greeksMembers);
232  }).ToList();
233 
234  var duplicateKeys = dataTypeMembers.GroupBy(x => x.Member.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
235  foreach (var duplicateKey in duplicateKeys)
236  {
237  throw new ArgumentException($"PandasData.ctor(): {Messages.PandasData.DuplicateKey(duplicateKey, type.FullName)}");
238  }
239 
240  // If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys
241  keys = dataTypeMembers.SelectMany(x => x.GetMemberNames()).ToHashSet();
242  keys.ExceptWith(_baseDataProperties);
243  keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type));
244  keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type));
245  keys.ExceptWith(GetPropertiesNames(typeof(Tick), type));
246  keys.Add("value");
247 
248  _members = dataTypeMembers.Where(x => x.GetMemberNames().All(name => keys.Contains(name))).ToList();
249  _membersByType.TryAdd(type, _members);
250  }
251  }
252 
253  var customColumns = new HashSet<string>(columns) { "value" };
254  customColumns.UnionWith(keys);
255 
256  columns = customColumns;
257  }
258 
259  _series = columns.ToDictionary(k => k, v => new Serie());
260  }
261 
262  /// <summary>
263  /// Adds security data object to the end of the lists
264  /// </summary>
265  /// <param name="baseData"><see cref="IBaseData"/> object that contains security data</param>
266  public void Add(object baseData)
267  {
268  var endTime = _isBaseData ? ((IBaseData)baseData).EndTime : default;
269  foreach (var member in _members)
270  {
271  if (!member.ShouldBeUnwrapped)
272  {
273  AddMemberToSeries(baseData, endTime, member.Member);
274  }
275  else
276  {
277  var memberValue = member.GetMemberValue(baseData);
278  if (memberValue != null)
279  {
280  foreach (var childMember in member.Children)
281  {
282  AddMemberToSeries(memberValue, endTime, childMember);
283  }
284  }
285  }
286  }
287 
288  var dynamicData = baseData as DynamicData;
289  var storage = dynamicData?.GetStorageDictionary();
290  if (storage != null)
291  {
292  var value = dynamicData.Value;
293  AddToSeries("value", endTime, value);
294 
295  foreach (var kvp in storage.Where(x => x.Key != "value"
296  // if this is a PythonData instance we add in '__typename' which we don't want into the data frame
297  && !x.Key.StartsWith("__", StringComparison.InvariantCulture)))
298  {
299  AddToSeries(kvp.Key, endTime, kvp.Value);
300  }
301  }
302  else if (baseData is Tick tick)
303  {
304  AddTick(tick);
305  }
306  else if (baseData is TradeBar tradeBar)
307  {
308  Add(tradeBar, null);
309  }
310  else if (baseData is QuoteBar quoteBar)
311  {
312  Add(null, quoteBar);
313  }
314  }
315 
316  private void AddMemberToSeries(object baseData, DateTime endTime, MemberInfo member)
317  {
318  // TODO field/property.GetValue is expensive
319  var key = member.Name.ToLowerInvariant();
320  if (member is PropertyInfo property)
321  {
322  var propertyValue = property.GetValue(baseData);
323  if (_isFundamentalType && property.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty)))
324  {
325  propertyValue = ((FundamentalTimeDependentProperty)propertyValue).Clone(new FixedTimeProvider(endTime));
326  }
327  AddToSeries(key, endTime, propertyValue);
328  }
329  else if (member is FieldInfo field)
330  {
331  AddToSeries(key, endTime, field.GetValue(baseData));
332  }
333  }
334 
335  /// <summary>
336  /// Adds Lean data objects to the end of the lists
337  /// </summary>
338  /// <param name="tradeBar"><see cref="TradeBar"/> object that contains trade bar information of the security</param>
339  /// <param name="quoteBar"><see cref="QuoteBar"/> object that contains quote bar information of the security</param>
340  public void Add(TradeBar tradeBar, QuoteBar quoteBar)
341  {
342  if (tradeBar != null)
343  {
344  var time = tradeBar.EndTime;
345  GetSerie(Open).Add(time, tradeBar.Open);
346  GetSerie(High).Add(time, tradeBar.High);
347  GetSerie(Low).Add(time, tradeBar.Low);
348  GetSerie(Close).Add(time, tradeBar.Close);
349  GetSerie(Volume).Add(time, tradeBar.Volume);
350  }
351  if (quoteBar != null)
352  {
353  var time = quoteBar.EndTime;
354  if (tradeBar == null)
355  {
356  GetSerie(Open).Add(time, quoteBar.Open);
357  GetSerie(High).Add(time, quoteBar.High);
358  GetSerie(Low).Add(time, quoteBar.Low);
359  GetSerie(Close).Add(time, quoteBar.Close);
360  }
361  if (quoteBar.Ask != null)
362  {
363  GetSerie(AskOpen).Add(time, quoteBar.Ask.Open);
364  GetSerie(AskHigh).Add(time, quoteBar.Ask.High);
365  GetSerie(AskLow).Add(time, quoteBar.Ask.Low);
366  GetSerie(AskClose).Add(time, quoteBar.Ask.Close);
367  GetSerie(AskSize).Add(time, quoteBar.LastAskSize);
368  }
369  if (quoteBar.Bid != null)
370  {
371  GetSerie(BidOpen).Add(time, quoteBar.Bid.Open);
372  GetSerie(BidHigh).Add(time, quoteBar.Bid.High);
373  GetSerie(BidLow).Add(time, quoteBar.Bid.Low);
374  GetSerie(BidClose).Add(time, quoteBar.Bid.Close);
375  GetSerie(BidSize).Add(time, quoteBar.LastBidSize);
376  }
377  }
378  }
379 
380  /// <summary>
381  /// Adds a tick data point to this pandas collection
382  /// </summary>
383  /// <param name="tick"><see cref="Tick"/> object that contains tick information of the security</param>
384  public void AddTick(Tick tick)
385  {
386  if (tick == null)
387  {
388  return;
389  }
390 
391  var time = tick.EndTime;
392 
393  // We will fill some series with null for tick types that don't have a value for that series, so that we make sure
394  // the indices are the same for every tick series.
395 
396  if (tick.TickType == TickType.Quote)
397  {
398  GetSerie(AskPrice).Add(time, tick.AskPrice);
399  GetSerie(AskSize).Add(time, tick.AskSize);
400  GetSerie(BidPrice).Add(time, tick.BidPrice);
401  GetSerie(BidSize).Add(time, tick.BidSize);
402  }
403  else
404  {
405  // Trade and open interest ticks don't have these values, so we'll fill them with null.
406  GetSerie(AskPrice).Add(time, null);
407  GetSerie(AskSize).Add(time, null);
408  GetSerie(BidPrice).Add(time, null);
409  GetSerie(BidSize).Add(time, null);
410  }
411 
412  GetSerie(Exchange).Add(time, tick.Exchange);
413  GetSerie(Suspicious).Add(time, tick.Suspicious);
414  GetSerie(Quantity).Add(time, tick.Quantity);
415 
416  if (tick.TickType == TickType.OpenInterest)
417  {
418  GetSerie(OpenInterest).Add(time, tick.Value);
419  GetSerie(LastPrice).Add(time, null);
420  }
421  else
422  {
423  GetSerie(LastPrice).Add(time, tick.Value);
424  GetSerie(OpenInterest).Add(time, null);
425  }
426  }
427 
428  /// <summary>
429  /// Get the pandas.DataFrame of the current <see cref="PandasData"/> state
430  /// </summary>
431  /// <param name="levels">Number of levels of the multi index</param>
432  /// <param name="filterMissingValueColumns">If false, make sure columns with "missing" values only are still added to the dataframe</param>
433  /// <returns>pandas.DataFrame object</returns>
434  public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns = true)
435  {
436  List<PyObject> list;
437  var symbol = _symbol.ToPython();
438 
439  // Create the index labels
440  var names = _defaultNames;
441 
442  if (levels == 1)
443  {
444  names = _level1Names;
445  list = new List<PyObject> { symbol };
446  }
447  else if (levels == 2)
448  {
449  // symbol, time
450  names = _level2Names;
451  list = new List<PyObject> { symbol, _empty };
452  }
453  else if (levels == 3)
454  {
455  // expiry, symbol, time
456  names = _level3Names;
457  list = new List<PyObject> { _symbol.ID.Date.ToPython(), symbol, _empty };
458  }
459  else
460  {
461  list = new List<PyObject> { _empty, _empty, _empty, symbol, _empty };
462  if (_symbol.SecurityType == SecurityType.Future)
463  {
464  list[0] = _symbol.ID.Date.ToPython();
465  }
466  else if (_symbol.SecurityType.IsOption())
467  {
468  list[0] = _symbol.ID.Date.ToPython();
469  list[1] = _symbol.ID.StrikePrice.ToPython();
470  list[2] = _symbol.ID.OptionRight.ToString().ToPython();
471  }
472  }
473 
474  // creating the pandas MultiIndex is expensive so we keep a cash
475  var indexCache = new Dictionary<List<DateTime>, PyObject>(new ListComparer<DateTime>());
476  // Returns a dictionary keyed by column name where values are pandas.Series objects
477  using var pyDict = new PyDict();
478  foreach (var kvp in _series)
479  {
480  if (filterMissingValueColumns && kvp.Value.ShouldFilter) continue;
481 
482  if (!indexCache.TryGetValue(kvp.Value.Times, out var index))
483  {
484  using var tuples = kvp.Value.Times.Select(time => CreateTupleIndex(time, list)).ToPyListUnSafe();
485  using var namesDic = Py.kw("names", names);
486 
487  indexCache[kvp.Value.Times] = index = _multiIndexFactory.Invoke(new[] { tuples }, namesDic);
488 
489  foreach (var pyObject in tuples)
490  {
491  pyObject.Dispose();
492  }
493  }
494 
495  // Adds pandas.Series value keyed by the column name
496  using var pyvalues = new PyList();
497  for (var i = 0; i < kvp.Value.Values.Count; i++)
498  {
499  using var pyObject = kvp.Value.Values[i].ToPython();
500  pyvalues.Append(pyObject);
501  }
502  using var series = _seriesFactory.Invoke(pyvalues, index);
503  using var pyStrKey = kvp.Key.ToPython();
504  using var pyKey = _pandasColumn.Invoke(pyStrKey);
505  pyDict.SetItem(pyKey, series);
506  }
507  _series.Clear();
508  foreach (var kvp in indexCache)
509  {
510  kvp.Value.Dispose();
511  }
512 
513  for (var i = 0; i < list.Count; i++)
514  {
515  DisposeIfNotEmpty(list[i]);
516  }
517 
518  // Create the DataFrame
519  var result = _dataFrameFactory.Invoke(pyDict);
520 
521  foreach (var item in pyDict)
522  {
523  item.Dispose();
524  }
525 
526  return result;
527  }
528 
529  /// <summary>
530  /// Helper method to create a single pandas data frame indexed by symbol
531  /// </summary>
532  /// <remarks>Will add a single point per pandas data series (symbol)</remarks>
533  public static PyObject ToPandasDataFrame(IEnumerable<PandasData> pandasDatas)
534  {
535  using var _ = Py.GIL();
536 
537  using var list = pandasDatas.Select(x => x._symbol).ToPyListUnSafe();
538 
539  using var namesDic = Py.kw("name", _level1Names[0]);
540  using var index = _indexFactory.Invoke(new[] { list }, namesDic);
541 
542  Dictionary<string, PyList> _valuesPerSeries = new();
543  foreach (var pandasData in pandasDatas)
544  {
545  foreach (var kvp in pandasData._series)
546  {
547  if (!_valuesPerSeries.TryGetValue(kvp.Key, out PyList value))
548  {
549  // Adds pandas.Series value keyed by the column name
550  value = _valuesPerSeries[kvp.Key] = new PyList();
551  }
552 
553  if (kvp.Value.Values.Count > 0)
554  {
555  // taking only 1 value per symbol
556  using var valueOfSymbol = kvp.Value.Values[0].ToPython();
557  value.Append(valueOfSymbol);
558  }
559  else
560  {
561  value.Append(PyObject.None);
562  }
563  }
564  }
565 
566  using var pyDict = new PyDict();
567  foreach (var kvp in _valuesPerSeries)
568  {
569  using var series = _seriesFactory.Invoke(kvp.Value, index);
570  using var pyStrKey = kvp.Key.ToPython();
571  using var pyKey = _pandasColumn.Invoke(pyStrKey);
572  pyDict.SetItem(pyKey, series);
573 
574  kvp.Value.Dispose();
575  }
576  var result = _dataFrameFactory.Invoke(pyDict);
577 
578  // Drop columns with only NaN or None values
579  using var dropnaKwargs = Py.kw("axis", 1, "inplace", true, "how", "all");
580  result.GetAttr("dropna").Invoke(Array.Empty<PyObject>(), dropnaKwargs);
581 
582  return result;
583  }
584 
585  /// <summary>
586  /// Only dipose of the PyObject if it was set to something different than empty
587  /// </summary>
588  private static void DisposeIfNotEmpty(PyObject pyObject)
589  {
590  if (!ReferenceEquals(pyObject, _empty))
591  {
592  pyObject.Dispose();
593  }
594  }
595 
596  /// <summary>
597  /// Create a new tuple index
598  /// </summary>
599  private static PyTuple CreateTupleIndex(DateTime index, List<PyObject> list)
600  {
601  if (list.Count > 1)
602  {
603  DisposeIfNotEmpty(list[list.Count - 1]);
604  list[list.Count - 1] = index.ToPython();
605  }
606  return new PyTuple(list.ToArray());
607  }
608 
609  /// <summary>
610  /// Adds data to dictionary
611  /// </summary>
612  /// <param name="key">The key of the value to get</param>
613  /// <param name="time"><see cref="DateTime"/> object to add to the value associated with the specific key</param>
614  /// <param name="input"><see cref="Object"/> to add to the value associated with the specific key. Can be null.</param>
615  private void AddToSeries(string key, DateTime time, object input)
616  {
617  var serie = GetSerie(key);
618  serie.Add(time, input);
619  }
620 
621  private Serie GetSerie(string key)
622  {
623  if (!_series.TryGetValue(key, out var value))
624  {
625  throw new ArgumentException($"PandasData.GetSerie(): {Messages.PandasData.KeyNotFoundInSeries(key)}");
626  }
627  return value;
628  }
629 
630  /// <summary>
631  /// Get the lower-invariant name of properties of the type that a another type is assignable from
632  /// </summary>
633  /// <param name="baseType">The type that is assignable from</param>
634  /// <param name="type">The type that is assignable by</param>
635  /// <returns>List of string. Empty list if not assignable from</returns>
636  private static IEnumerable<string> GetPropertiesNames(Type baseType, Type type)
637  {
638  return baseType.IsAssignableFrom(type)
639  ? baseType.GetProperties().Select(x => x.Name.ToLowerInvariant())
640  : Enumerable.Empty<string>();
641  }
642 
643  private class Serie
644  {
645  private static readonly IFormatProvider InvariantCulture = CultureInfo.InvariantCulture;
646  public bool ShouldFilter { get; set; } = true;
647  public List<DateTime> Times { get; set; } = new();
648  public List<object> Values { get; set; } = new();
649 
650  public void Add(DateTime time, object input)
651  {
652  var value = input is decimal ? Convert.ToDouble(input, InvariantCulture) : input;
653  if (ShouldFilter)
654  {
655  // we need at least 1 valid entry for the series not to get filtered
656  if (value is double)
657  {
658  if (!((double)value).IsNaNOrZero())
659  {
660  ShouldFilter = false;
661  }
662  }
663  else if (value is string)
664  {
665  if (!string.IsNullOrWhiteSpace((string)value))
666  {
667  ShouldFilter = false;
668  }
669  }
670  else if (value is bool)
671  {
672  if ((bool)value)
673  {
674  ShouldFilter = false;
675  }
676  }
677  else if (value != null)
678  {
679  ShouldFilter = false;
680  }
681  }
682 
683  Values.Add(value);
684  Times.Add(time);
685  }
686 
687  public void Add(DateTime time, decimal input)
688  {
689  var value = Convert.ToDouble(input, InvariantCulture);
690  if (ShouldFilter && !value.IsNaNOrZero())
691  {
692  ShouldFilter = false;
693  }
694 
695  Values.Add(value);
696  Times.Add(time);
697  }
698  }
699 
700  private class FixedTimeProvider : ITimeProvider
701  {
702  private readonly DateTime _time;
703  public DateTime GetUtcNow() => _time;
704  public FixedTimeProvider(DateTime time)
705  {
706  _time = time;
707  }
708  }
709 
710  private class DataTypeMember
711  {
712  public MemberInfo Member { get; }
713 
714  public MemberInfo[] Children { get; }
715 
716  public bool ShouldBeUnwrapped => Children != null && Children.Length > 0;
717 
718  public DataTypeMember(MemberInfo member, MemberInfo[] children = null)
719  {
720  Member = member;
721  Children = children;
722  }
723 
724  public IEnumerable<string> GetMemberNames()
725  {
726  // If there are no children, return the name of the member. Else ignore the member and return the children names
727  if (ShouldBeUnwrapped)
728  {
729  foreach (var child in Children)
730  {
731  yield return child.Name.ToLowerInvariant();
732  }
733  yield break;
734  }
735 
736  yield return Member.Name.ToLowerInvariant();
737  }
738 
739  public object GetMemberValue(object instance)
740  {
741  return Member switch
742  {
743  PropertyInfo property => property.GetValue(instance),
744  FieldInfo field => field.GetValue(instance),
745  // Should not happen
746  _ => throw new InvalidOperationException($"Unexpected member type: {Member.MemberType}")
747  };
748  }
749 
750  public static Type GetMemberType(MemberInfo member)
751  {
752  return member switch
753  {
754  PropertyInfo property => property.PropertyType,
755  FieldInfo field => field.FieldType,
756  // Should not happen
757  _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}")
758  };
759  }
760  }
761  }
762 }