22 using System.Collections;
23 using System.Collections.Concurrent;
24 using System.Collections.Generic;
25 using System.Globalization;
27 using System.Reflection;
36 private const string Open =
"open";
37 private const string High =
"high";
38 private const string Low =
"low";
39 private const string Close =
"close";
40 private const string Volume =
"volume";
42 private const string AskOpen =
"askopen";
43 private const string AskHigh =
"askhigh";
44 private const string AskLow =
"asklow";
45 private const string AskClose =
"askclose";
46 private const string AskPrice =
"askprice";
47 private const string AskSize =
"asksize";
49 private const string BidOpen =
"bidopen";
50 private const string BidHigh =
"bidhigh";
51 private const string BidLow =
"bidlow";
52 private const string BidClose =
"bidclose";
53 private const string BidPrice =
"bidprice";
54 private const string BidSize =
"bidsize";
56 private const string LastPrice =
"lastprice";
57 private const string Quantity =
"quantity";
58 private const string Exchange =
"exchange";
59 private const string Suspicious =
"suspicious";
62 #region OptionContract Members Handling
68 private static readonly
string[] _optionContractExcludedMembers =
new[]
73 private static readonly
string[] _greeksMemberNames =
new[]
82 private static readonly MemberInfo[] _greeksMembers = typeof(
Greeks)
83 .GetMembers(BindingFlags.Instance | BindingFlags.Public)
84 .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) &&
85 _greeksMemberNames.Contains(x.Name.ToLowerInvariant()))
91 private static PyString _empty;
92 private static PyObject _pandas;
93 private static PyObject _pandasColumn;
94 private static PyObject _seriesFactory;
95 private static PyObject _dataFrameFactory;
96 private static PyObject _multiIndexFactory;
97 private static PyObject _multiIndex;
98 private static PyObject _indexFactory;
100 private static PyList _defaultNames;
101 private static PyList _level1Names;
102 private static PyList _level2Names;
103 private static PyList _level3Names;
105 private readonly
static HashSet<string> _baseDataProperties = typeof(
BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant());
106 private readonly
static ConcurrentDictionary<Type, IEnumerable<DataTypeMember>> _membersByType =
new();
107 private readonly
static IReadOnlyList<string> _standardColumns =
new string[]
109 Open, High, Low, Close, LastPrice, Volume,
110 AskOpen, AskHigh, AskLow, AskClose, AskPrice, AskSize, Quantity, Suspicious,
114 private readonly
Symbol _symbol;
115 private readonly
bool _isFundamentalType;
116 private readonly
bool _isBaseData;
117 private readonly Dictionary<string, Serie> _series;
119 private readonly IEnumerable<DataTypeMember> _members = Enumerable.
Empty<DataTypeMember>();
139 _pandas = Py.Import(
"PandasMapper");
140 _pandasColumn = _pandas.GetAttr(
"PandasColumn");
141 _seriesFactory = _pandas.GetAttr(
"Series");
142 _dataFrameFactory = _pandas.GetAttr(
"DataFrame");
143 _multiIndex = _pandas.GetAttr(
"MultiIndex");
144 _multiIndexFactory = _multiIndex.GetAttr(
"from_tuples");
145 _indexFactory = _pandas.GetAttr(
"Index");
146 _empty =
new PyString(
string.Empty);
148 var time =
new PyString(
"time");
149 var symbol =
new PyString(
"symbol");
150 var expiry =
new PyString(
"expiry");
151 _defaultNames =
new PyList(
new PyObject[] { expiry,
new PyString(
"strike"),
new PyString(
"type"), symbol, time });
152 _level1Names =
new PyList(
new PyObject[] { symbol });
153 _level2Names =
new PyList(
new PyObject[] { symbol, time });
154 _level3Names =
new PyList(
new PyObject[] { expiry, symbol, time });
167 if (baseData ==
null && data is IEnumerable enumerable)
169 foreach (var item
in enumerable)
177 var type = data.GetType();
179 _isBaseData = baseData !=
null;
180 _symbol = _isBaseData ? baseData.Symbol : ((
ISymbolProvider)data).Symbol;
183 if (baseData ==
null)
191 else if (_symbol.SecurityType.IsOption())
196 IEnumerable<string> columns = _standardColumns;
200 var keys = (data as
DynamicData)?.GetStorageDictionary()
202 .Where(x => !x.Key.StartsWith(
"__", StringComparison.InvariantCulture)).ToHashSet(x => x.Key);
207 if (_membersByType.TryGetValue(type, out _members))
209 keys = _members.SelectMany(x => x.GetMemberNames()).ToHashSet();
214 .GetMembers(BindingFlags.Instance | BindingFlags.Public)
215 .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property);
221 members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name));
224 var dataTypeMembers = members.Select(x =>
226 if (!DataTypeMember.GetMemberType(x).IsAssignableTo(typeof(
Greeks)))
228 return new DataTypeMember(x);
231 return new DataTypeMember(x, _greeksMembers);
234 var duplicateKeys = dataTypeMembers.GroupBy(x => x.Member.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
235 foreach (var duplicateKey
in duplicateKeys)
237 throw new ArgumentException($
"PandasData.ctor(): {Messages.PandasData.DuplicateKey(duplicateKey, type.FullName)}");
241 keys = dataTypeMembers.SelectMany(x => x.GetMemberNames()).ToHashSet();
242 keys.ExceptWith(_baseDataProperties);
243 keys.ExceptWith(GetPropertiesNames(typeof(
QuoteBar), type));
244 keys.ExceptWith(GetPropertiesNames(typeof(
TradeBar), type));
245 keys.ExceptWith(GetPropertiesNames(typeof(
Tick), type));
248 _members = dataTypeMembers.Where(x => x.GetMemberNames().All(name => keys.Contains(name))).ToList();
249 _membersByType.TryAdd(type, _members);
253 var customColumns =
new HashSet<string>(columns) {
"value" };
254 customColumns.UnionWith(keys);
256 columns = customColumns;
259 _series = columns.ToDictionary(k => k, v =>
new Serie());
266 public void Add(
object baseData)
268 var endTime = _isBaseData ? ((
IBaseData)baseData).EndTime :
default;
269 foreach (var member
in _members)
271 if (!member.ShouldBeUnwrapped)
273 AddMemberToSeries(baseData, endTime, member.Member);
277 var memberValue = member.GetMemberValue(baseData);
278 if (memberValue !=
null)
280 foreach (var childMember
in member.Children)
282 AddMemberToSeries(memberValue, endTime, childMember);
292 var value = dynamicData.Value;
293 AddToSeries(
"value", endTime, value);
295 foreach (var kvp
in storage.Where(x => x.Key !=
"value"
297 && !x.Key.StartsWith(
"__", StringComparison.InvariantCulture)))
299 AddToSeries(kvp.Key, endTime, kvp.Value);
302 else if (baseData is
Tick tick)
306 else if (baseData is
TradeBar tradeBar)
310 else if (baseData is
QuoteBar quoteBar)
316 private void AddMemberToSeries(
object baseData, DateTime endTime, MemberInfo member)
319 var key = member.Name.ToLowerInvariant();
320 if (member is PropertyInfo property)
322 var propertyValue =
property.GetValue(baseData);
327 AddToSeries(key, endTime, propertyValue);
329 else if (member is FieldInfo field)
331 AddToSeries(key, endTime, field.GetValue(baseData));
342 if (tradeBar !=
null)
345 GetSerie(Open).Add(time, tradeBar.
Open);
346 GetSerie(High).Add(time, tradeBar.
High);
347 GetSerie(Low).Add(time, tradeBar.
Low);
348 GetSerie(Close).Add(time, tradeBar.
Close);
349 GetSerie(Volume).Add(time, tradeBar.
Volume);
351 if (quoteBar !=
null)
354 if (tradeBar ==
null)
356 GetSerie(Open).Add(time, quoteBar.
Open);
357 GetSerie(High).Add(time, quoteBar.
High);
358 GetSerie(Low).Add(time, quoteBar.
Low);
359 GetSerie(Close).Add(time, quoteBar.
Close);
361 if (quoteBar.
Ask !=
null)
363 GetSerie(AskOpen).Add(time, quoteBar.
Ask.
Open);
364 GetSerie(AskHigh).Add(time, quoteBar.
Ask.
High);
365 GetSerie(AskLow).Add(time, quoteBar.
Ask.
Low);
366 GetSerie(AskClose).Add(time, quoteBar.
Ask.
Close);
369 if (quoteBar.
Bid !=
null)
371 GetSerie(BidOpen).Add(time, quoteBar.
Bid.
Open);
372 GetSerie(BidHigh).Add(time, quoteBar.
Bid.
High);
373 GetSerie(BidLow).Add(time, quoteBar.
Bid.
Low);
374 GetSerie(BidClose).Add(time, quoteBar.
Bid.
Close);
398 GetSerie(AskPrice).Add(time, tick.
AskPrice);
399 GetSerie(AskSize).Add(time, tick.
AskSize);
400 GetSerie(BidPrice).Add(time, tick.
BidPrice);
401 GetSerie(BidSize).Add(time, tick.
BidSize);
406 GetSerie(AskPrice).Add(time,
null);
407 GetSerie(AskSize).Add(time,
null);
408 GetSerie(BidPrice).Add(time,
null);
409 GetSerie(BidSize).Add(time,
null);
413 GetSerie(Suspicious).Add(time, tick.
Suspicious);
414 GetSerie(Quantity).Add(time, tick.
Quantity);
419 GetSerie(LastPrice).Add(time,
null);
423 GetSerie(LastPrice).Add(time, tick.
Value);
437 var symbol = _symbol.ToPython();
440 var names = _defaultNames;
444 names = _level1Names;
445 list =
new List<PyObject> { symbol };
447 else if (levels == 2)
450 names = _level2Names;
451 list =
new List<PyObject> { symbol, _empty };
453 else if (levels == 3)
456 names = _level3Names;
457 list =
new List<PyObject> { _symbol.ID.Date.ToPython(), symbol, _empty };
461 list =
new List<PyObject> { _empty, _empty, _empty, symbol, _empty };
464 list[0] = _symbol.ID.Date.ToPython();
466 else if (_symbol.SecurityType.IsOption())
468 list[0] = _symbol.ID.Date.ToPython();
469 list[1] = _symbol.ID.StrikePrice.ToPython();
470 list[2] = _symbol.ID.OptionRight.ToString().ToPython();
477 using var pyDict =
new PyDict();
478 foreach (var kvp
in _series)
480 if (filterMissingValueColumns && kvp.Value.ShouldFilter)
continue;
482 if (!indexCache.TryGetValue(kvp.Value.Times, out var index))
484 using var tuples = kvp.Value.Times.Select(time => CreateTupleIndex(time, list)).ToPyListUnSafe();
485 using var namesDic = Py.kw(
"names", names);
487 indexCache[kvp.Value.Times] = index = _multiIndexFactory.Invoke(
new[] { tuples }, namesDic);
489 foreach (var pyObject
in tuples)
496 using var pyvalues =
new PyList();
497 for (var i = 0; i < kvp.Value.Values.Count; i++)
499 using var pyObject = kvp.Value.Values[i].ToPython();
500 pyvalues.Append(pyObject);
502 using var series = _seriesFactory.Invoke(pyvalues, index);
503 using var pyStrKey = kvp.Key.ToPython();
504 using var pyKey = _pandasColumn.Invoke(pyStrKey);
505 pyDict.SetItem(pyKey, series);
508 foreach (var kvp
in indexCache)
513 for (var i = 0; i < list.Count; i++)
515 DisposeIfNotEmpty(list[i]);
519 var result = _dataFrameFactory.Invoke(pyDict);
521 foreach (var item
in pyDict)
535 using var _ = Py.GIL();
537 using var list = pandasDatas.Select(x => x._symbol).ToPyListUnSafe();
539 using var namesDic = Py.kw(
"name", _level1Names[0]);
540 using var index = _indexFactory.Invoke(
new[] { list }, namesDic);
542 Dictionary<string, PyList> _valuesPerSeries =
new();
543 foreach (var pandasData
in pandasDatas)
545 foreach (var kvp
in pandasData._series)
547 if (!_valuesPerSeries.TryGetValue(kvp.Key, out PyList value))
550 value = _valuesPerSeries[kvp.Key] =
new PyList();
553 if (kvp.Value.Values.Count > 0)
556 using var valueOfSymbol = kvp.Value.Values[0].ToPython();
557 value.Append(valueOfSymbol);
561 value.Append(PyObject.None);
566 using var pyDict =
new PyDict();
567 foreach (var kvp
in _valuesPerSeries)
569 using var series = _seriesFactory.Invoke(kvp.Value, index);
570 using var pyStrKey = kvp.Key.ToPython();
571 using var pyKey = _pandasColumn.Invoke(pyStrKey);
572 pyDict.SetItem(pyKey, series);
576 var result = _dataFrameFactory.Invoke(pyDict);
579 using var dropnaKwargs = Py.kw(
"axis", 1,
"inplace",
true,
"how",
"all");
580 result.GetAttr(
"dropna").Invoke(Array.Empty<PyObject>(), dropnaKwargs);
588 private static void DisposeIfNotEmpty(PyObject pyObject)
590 if (!ReferenceEquals(pyObject, _empty))
599 private static PyTuple CreateTupleIndex(DateTime index, List<PyObject> list)
603 DisposeIfNotEmpty(list[list.Count - 1]);
604 list[list.Count - 1] = index.ToPython();
606 return new PyTuple(list.ToArray());
615 private void AddToSeries(
string key, DateTime time,
object input)
617 var serie = GetSerie(key);
618 serie.Add(time, input);
621 private Serie GetSerie(
string key)
623 if (!_series.TryGetValue(key, out var value))
625 throw new ArgumentException($
"PandasData.GetSerie(): {Messages.PandasData.KeyNotFoundInSeries(key)}");
636 private static IEnumerable<string> GetPropertiesNames(Type baseType, Type type)
638 return baseType.IsAssignableFrom(type)
639 ? baseType.GetProperties().Select(x => x.Name.ToLowerInvariant())
640 : Enumerable.Empty<
string>();
645 private static readonly IFormatProvider InvariantCulture = CultureInfo.InvariantCulture;
646 public bool ShouldFilter {
get;
set; } =
true;
647 public List<DateTime> Times {
get;
set; } =
new();
648 public List<object> Values {
get;
set; } =
new();
650 public void Add(DateTime time,
object input)
652 var value = input is decimal ? Convert.ToDouble(input, InvariantCulture) : input;
658 if (!((
double)value).IsNaNOrZero())
660 ShouldFilter =
false;
663 else if (value is
string)
665 if (!
string.IsNullOrWhiteSpace((
string)value))
667 ShouldFilter =
false;
670 else if (value is
bool)
674 ShouldFilter =
false;
677 else if (value !=
null)
679 ShouldFilter =
false;
687 public void Add(DateTime time, decimal input)
689 var value = Convert.ToDouble(input, InvariantCulture);
690 if (ShouldFilter && !value.IsNaNOrZero())
692 ShouldFilter =
false;
700 private class FixedTimeProvider : ITimeProvider
702 private readonly DateTime _time;
703 public DateTime GetUtcNow() => _time;
704 public FixedTimeProvider(DateTime time)
710 private class DataTypeMember
712 public MemberInfo Member {
get; }
714 public MemberInfo[] Children {
get; }
716 public bool ShouldBeUnwrapped => Children !=
null && Children.Length > 0;
718 public DataTypeMember(MemberInfo member, MemberInfo[] children =
null)
724 public IEnumerable<string> GetMemberNames()
727 if (ShouldBeUnwrapped)
729 foreach (var child
in Children)
731 yield
return child.Name.ToLowerInvariant();
736 yield
return Member.Name.ToLowerInvariant();
739 public object GetMemberValue(
object instance)
743 PropertyInfo
property =>
property.GetValue(instance),
744 FieldInfo field => field.GetValue(instance),
746 _ =>
throw new InvalidOperationException($
"Unexpected member type: {Member.MemberType}")
750 public static Type GetMemberType(MemberInfo member)
754 PropertyInfo
property =>
property.PropertyType,
755 FieldInfo field => field.FieldType,
757 _ =>
throw new InvalidOperationException($
"Unexpected member type: {member.MemberType}")