17 using System.Collections.Generic;
38 private readonly Dictionary<DateTime, Dictionary<Symbol, Security>> _pendingSecurityAdditions =
new Dictionary<DateTime, Dictionary<Symbol, Security>>();
42 private bool _initializedSecurityBenchmark;
43 private bool _anyDoesNotHaveFundamentalDataWarningLogged;
61 _algorithm = algorithm;
62 _securityService = securityService;
79 if (_dataManager !=
null)
81 throw new Exception(
"UniverseSelection.SetDataManager(): can only be set once");
83 _dataManager = dataManager;
85 _internalSubscriptionManager.Added += (sender, request) =>
89 _internalSubscriptionManager.Removed += (sender, request) =>
103 var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
104 if (dateTimeUtc > algorithmEndDateUtc)
109 IEnumerable<Symbol> selectSymbolsResult;
115 if (fineFiltered !=
null
122 selectSymbolsResult = universe.
SelectSymbols(dateTimeUtc, universeData);
130 var hasFundamentalData = universeData.
Data.Count > 0 && universeData.
Data[0] is
Fundamental;
131 if(hasFundamentalData)
134 var anyDoesNotHaveFundamentalData =
false;
139 var
set = selectSymbolsResult.ToHashSet();
140 fineCollection.Data.AddRange(universeData.
Data.OfType<
Fundamental>().Where(fundamental => {
142 if (set.Remove(fundamental.Symbol))
144 if (!fundamental.HasFundamentalData)
146 anyDoesNotHaveFundamentalData = true;
154 if (!_anyDoesNotHaveFundamentalDataWarningLogged && anyDoesNotHaveFundamentalData)
156 _algorithm.Debug(
"Note: Your coarse selection filter was updated to exclude symbols without fine fundamental data. Make sure your coarse filter excludes symbols where HasFundamental is false.");
157 _anyDoesNotHaveFundamentalDataWarningLogged =
true;
164 foreach (var symbol
in selectSymbolsResult)
166 fineCollection.Data.Add(
new Fundamental(currentTime, symbol));
170 universeData.
Data = fineCollection.Data;
172 selectSymbolsResult = fineFiltered.
PerformSelection(dateTimeUtc, fineCollection);
178 selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
184 universe.Selected = selectSymbolsResult.ToHashSet();
189 RemoveSecurityFromUniverse(
190 _pendingRemovalsManager.CheckPendingRemovals(universe.Selected, universe),
192 algorithmEndDateUtc);
201 foreach (var member
in universe.Securities.Values.OrderBy(member => member.Security.Symbol.SecurityType).ThenBy(x => x.Security.Symbol.ID))
203 var security = member.Security;
205 if (universe.Selected.Contains(security.Symbol))
continue;
208 if (!universe.CanRemoveMember(dateTimeUtc, security))
continue;
210 if (!member.Security.IsDelisted)
213 _securityChangesConstructor.Remove(member.Security, member.IsInternal);
216 RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(security, universe),
218 algorithmEndDateUtc);
221 Dictionary<Symbol, Security> pendingAdditions;
222 if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
225 _pendingSecurityAdditions.Clear();
228 pendingAdditions =
new Dictionary<Symbol, Security>();
229 _pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
233 foreach (var symbol
in universe.Selected)
235 if (universe.Securities.ContainsKey(symbol))
242 if (symbol.HasUnderlying)
244 underlying = GetOrCreateSecurity(pendingAdditions, symbol.Underlying, universe.UniverseSettings);
247 var security = GetOrCreateSecurity(pendingAdditions, symbol, universe.UniverseSettings, underlying);
249 var addedSubscription =
false;
250 var dataFeedAdded =
false;
251 var internalFeed =
true;
252 foreach (var request
in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc,
253 _algorithm.SubscriptionManager.SubscriptionDataConfigService))
255 if (security.Symbol == request.Configuration.Symbol
256 && !security.Subscriptions.Contains(request.Configuration))
259 security.AddData(request.Configuration);
262 var toRemove = _currencySubscriptionDataConfigManager.GetSubscriptionDataConfigToRemove(request.Configuration.Symbol);
263 if (toRemove !=
null)
265 Log.
Trace($
"UniverseSelection.ApplyUniverseSelection(): Removing internal currency data feed {toRemove}");
274 if (!request.IsUniverseSubscription)
276 addedSubscription =
true;
278 internalFeed &= request.Configuration.IsInternalFeed;
279 _internalSubscriptionManager.AddedSubscriptionRequest(request);
283 if (addedSubscription)
285 var addedMember = universe.AddMember(dateTimeUtc, security, internalFeed);
287 if (addedMember && dataFeedAdded)
289 _securityChangesConstructor.Add(security, internalFeed);
294 var securityChanges = _securityChangesConstructor.Flush();
297 if (securityChanges.AddedSecurities.Count > 0)
306 Log.
Debug(
"UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc +
": " + securityChanges);
309 return securityChanges;
320 if (!_initializedSecurityBenchmark)
322 _initializedSecurityBenchmark =
true;
325 if (securityBenchmark !=
null)
330 var subscriptionType = _algorithm.SubscriptionManager.SubscriptionDataConfigService.LookupSubscriptionConfigDataTypes(securityBenchmark.Security.Type, resolution, securityBenchmark.Security.Symbol.IsCanonical()).First();
332 var isCustomData =
false;
338 var symbolDataConfigs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(symbol);
339 if (symbolDataConfigs.Any())
341 subscriptionType =
new Tuple<Type, TickType>(symbolDataConfigs.First().Type,
TickType.Trade);
346 var baseInstance = subscriptionType.Item1.GetBaseDataInstance();
347 baseInstance.Symbol = securityBenchmark.Security.Symbol;
348 var supportedResolutions = baseInstance.SupportedResolutions();
349 if (!supportedResolutions.Contains(resolution))
351 resolution = supportedResolutions.OrderByDescending(x => x).First();
354 var subscriptionList =
new List<Tuple<Type, TickType>>() {subscriptionType};
355 var dataConfig = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(
356 securityBenchmark.Security.Symbol,
358 isInternalFeed:
true,
360 isCustomData: isCustomData,
361 subscriptionDataTypes: subscriptionList
367 securityBenchmark.Security.Exchange.Hours,
368 utcStart.ConvertFromUtc(securityBenchmark.Security.Exchange.TimeZone),
372 dataConfig.DataTimeZone,
374 ).ConvertToUtc(securityBenchmark.Security.Exchange.TimeZone);
376 if (dataConfig !=
null)
381 securityBenchmark.Security,
384 _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)));
386 Log.
Trace($
"UniverseSelection.AddPendingInternalDataFeeds(): Adding internal benchmark data feed {dataConfig}");
391 if (_currencySubscriptionDataConfigManager.UpdatePendingSubscriptionDataConfigs(_algorithm.BrokerageModel))
393 foreach (var subscriptionDataConfig
in _currencySubscriptionDataConfigManager
394 .GetPendingSubscriptionDataConfigs())
396 var security = _algorithm.Securities[subscriptionDataConfig.Symbol];
401 subscriptionDataConfig,
403 _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)));
414 _currencySubscriptionDataConfigManager.EnsureCurrencySubscriptionDataConfigs(securityChanges, _algorithm.BrokerageModel);
422 if (_algorithm.Securities.TryGetValue(data.
Symbol, out var security))
425 security.IsDelisted =
true;
426 security.IsTradable =
false;
430 if (_algorithm.Securities.Remove(data.
Symbol) && !_pendingRemovalsManager.PendingRemovals.Values.Any(x => x.Any(y => y.Symbol == data.
Symbol)))
432 _securityChangesConstructor.Remove(security, isInternalFeed);
434 return _securityChangesConstructor.Flush();
441 private void RemoveSecurityFromUniverse(
442 List<PendingRemovalsManager.RemovedMember> removedMembers,
443 DateTime dateTimeUtc,
444 DateTime algorithmEndDateUtc)
446 if (removedMembers ==
null)
450 foreach (var removedMember
in removedMembers)
452 var universe = removedMember.Universe;
453 var member = removedMember.Security;
456 universe.RemoveMember(dateTimeUtc, member);
458 var isActive = _algorithm.UniverseManager.ActiveSecurities.ContainsKey(member.Symbol);
459 foreach (var subscription
in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc,
460 _algorithm.SubscriptionManager.SubscriptionDataConfigService))
464 _internalSubscriptionManager.RemovedSubscriptionRequest(subscription);
469 member.IsTradable =
false;
474 member.Cache.Reset();
476 _algorithm.Securities.Remove(member.Symbol);
483 private Security GetOrCreateSecurity(Dictionary<Symbol, Security> pendingAdditions, Symbol symbol,
UniverseSettings universeSettings,
Security underlying =
null)
487 if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
489 security = _securityService.CreateSecurity(symbol,
new List<SubscriptionDataConfig>(), universeSettings.
Leverage, symbol.ID.SecurityType.IsOption(), underlying);
491 pendingAdditions.
Add(symbol, security);