Lean  $LEAN_TAG$
Config.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using System;
17 using System.Collections.Generic;
18 using System.Globalization;
19 using System.IO;
20 using System.Linq;
21 using Newtonsoft.Json;
22 using Newtonsoft.Json.Linq;
23 using QuantConnect.Logging;
24 using static System.FormattableString;
25 
27 {
28  /// <summary>
29  /// Configuration class loads the required external setup variables to launch the Lean engine.
30  /// </summary>
31  public static class Config
32  {
33  //Location of the configuration file.
34  private static string ConfigurationFileName = "config.json";
35 
36  /// <summary>
37  /// Set configuration file on-fly
38  /// </summary>
39  /// <param name="fileName"></param>
40  public static void SetConfigurationFile(string fileName)
41  {
42  if (File.Exists(fileName))
43  {
44  Log.Trace(Invariant($"Using {fileName} as configuration file"));
45  ConfigurationFileName = fileName;
46  }
47  else
48  {
49  Log.Error(Invariant($"Configuration file {fileName} does not exist, using {ConfigurationFileName}"));
50  }
51  }
52 
53  /// <summary>
54  /// Merge CLI arguments with configuration file + load custom config file via CLI arg
55  /// </summary>
56  /// <param name="cliArguments"></param>
57  public static void MergeCommandLineArgumentsWithConfiguration(Dictionary<string, object> cliArguments)
58  {
59  if (cliArguments.ContainsKey("config"))
60  {
61  SetConfigurationFile(cliArguments["config"] as string);
62  Reset();
63  }
64 
65  var jsonArguments = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(cliArguments));
66 
67  Settings.Value.Merge(jsonArguments, new JsonMergeSettings
68  {
69  MergeArrayHandling = MergeArrayHandling.Union
70  });
71  }
72 
73  /// <summary>
74  /// Resets the config settings to their default values.
75  /// Called in regression tests where multiple algorithms are run sequentially,
76  /// and we need to guarantee that every test starts with the same configuration.
77  /// </summary>
78  public static void Reset()
79  {
80  Settings = new Lazy<JObject>(ConfigFactory);
81  }
82 
83  private static Lazy<JObject> Settings = new Lazy<JObject>(ConfigFactory);
84 
85  private static JObject ConfigFactory()
86  {
87  // initialize settings inside a lazy for free thread-safe, one-time initialization
88  if (!File.Exists(ConfigurationFileName))
89  {
90  return new JObject
91  {
92  {"algorithm-type-name", "BasicTemplateAlgorithm"},
93  {"live-mode", false},
94  {"data-folder", "../../../Data/"},
95  {"messaging-handler", "QuantConnect.Messaging.Messaging"},
96  {"job-queue-handler", "QuantConnect.Queues.JobQueue"},
97  {"api-handler", "QuantConnect.Api.Api"},
98  {"setup-handler", "QuantConnect.Lean.Engine.Setup.ConsoleSetupHandler"},
99  {"result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler"},
100  {"data-feed-handler", "QuantConnect.Lean.Engine.DataFeeds.FileSystemDataFeed"},
101  {"real-time-handler", "QuantConnect.Lean.Engine.RealTime.BacktestingRealTimeHandler"},
102  {"transaction-handler", "QuantConnect.Lean.Engine.TransactionHandlers.BacktestingTransactionHandler"}
103  };
104  }
105 
106  return JObject.Parse(File.ReadAllText(ConfigurationFileName));
107  }
108 
109  /// <summary>
110  /// Gets the currently selected environment. If sub-environments are defined,
111  /// they'll be returned as {env1}.{env2}
112  /// </summary>
113  /// <returns>The fully qualified currently selected environment</returns>
114  public static string GetEnvironment()
115  {
116  var environments = new List<string>();
117  JToken currentEnvironment = Settings.Value;
118  var env = currentEnvironment["environment"];
119  while (currentEnvironment != null && env != null)
120  {
121  var currentEnv = env.Value<string>();
122  environments.Add(currentEnv);
123  var moreEnvironments = currentEnvironment["environments"];
124  if (moreEnvironments == null)
125  {
126  break;
127  }
128 
129  currentEnvironment = moreEnvironments[currentEnv];
130  env = currentEnvironment["environment"];
131  }
132  return string.Join(".", environments);
133  }
134 
135  /// <summary>
136  /// Get the matching config setting from the file searching for this key.
137  /// </summary>
138  /// <param name="key">String key value we're seaching for in the config file.</param>
139  /// <param name="defaultValue"></param>
140  /// <returns>String value of the configuration setting or empty string if nothing found.</returns>
141  public static string Get(string key, string defaultValue = "")
142  {
143  // special case environment requests
144  if (key == "environment") return GetEnvironment();
145 
146  var token = GetToken(Settings.Value, key);
147  if (token == null)
148  {
149  Log.Trace(Invariant($"Config.Get(): Configuration key not found. Key: {key} - Using default value: {defaultValue}"));
150  return defaultValue;
151  }
152  return token.ToString();
153  }
154 
155  /// <summary>
156  /// Gets the underlying JToken for the specified key
157  /// </summary>
158  public static JToken GetToken(string key)
159  {
160  return GetToken(Settings.Value, key);
161  }
162 
163  /// <summary>
164  /// Sets a configuration value. This is really only used to help testing. The key heye can be
165  /// specified as {environment}.key to set a value on a specific environment
166  /// </summary>
167  /// <param name="key">The key to be set</param>
168  /// <param name="value">The new value</param>
169  public static void Set(string key, dynamic value)
170  {
171  JToken environment = Settings.Value;
172  while (key.Contains('.', StringComparison.InvariantCulture))
173  {
174  var envName = key.Substring(0, key.IndexOf(".", StringComparison.InvariantCulture));
175  key = key.Substring(key.IndexOf(".", StringComparison.InvariantCulture) + 1);
176  var environments = environment["environments"];
177  if (environments == null)
178  {
179  environment["environments"] = environments = new JObject();
180  }
181  environment = environments[envName];
182  }
183  environment[key] = value;
184  }
185 
186  /// <summary>
187  /// Get a boolean value configuration setting by a configuration key.
188  /// </summary>
189  /// <param name="key">String value of the configuration key.</param>
190  /// <param name="defaultValue">The default value to use if not found in configuration</param>
191  /// <returns>Boolean value of the config setting.</returns>
192  public static bool GetBool(string key, bool defaultValue = false)
193  {
194  return GetValue(key, defaultValue);
195  }
196 
197  /// <summary>
198  /// Get the int value of a config string.
199  /// </summary>
200  /// <param name="key">Search key from the config file</param>
201  /// <param name="defaultValue">The default value to use if not found in configuration</param>
202  /// <returns>Int value of the config setting.</returns>
203  public static int GetInt(string key, int defaultValue = 0)
204  {
205  return GetValue(key, defaultValue);
206  }
207 
208  /// <summary>
209  /// Get the double value of a config string.
210  /// </summary>
211  /// <param name="key">Search key from the config file</param>
212  /// <param name="defaultValue">The default value to use if not found in configuration</param>
213  /// <returns>Double value of the config setting.</returns>
214  public static double GetDouble(string key, double defaultValue = 0.0)
215  {
216  return GetValue(key, defaultValue);
217  }
218 
219  /// <summary>
220  /// Gets a value from configuration and converts it to the requested type, assigning a default if
221  /// the configuration is null or empty
222  /// </summary>
223  /// <typeparam name="T">The requested type</typeparam>
224  /// <param name="key">Search key from the config file</param>
225  /// <param name="defaultValue">The default value to use if not found in configuration</param>
226  /// <returns>Converted value of the config setting.</returns>
227  public static T GetValue<T>(string key, T defaultValue = default(T))
228  {
229  // special case environment requests
230  if (key == "environment" && typeof (T) == typeof (string)) return (T) (object) GetEnvironment();
231 
232  var token = GetToken(Settings.Value, key);
233  if (token == null)
234  {
235  var defaultValueString = defaultValue is IConvertible
236  ? ((IConvertible) defaultValue).ToString(CultureInfo.InvariantCulture)
237  : defaultValue is IFormattable
238  ? ((IFormattable) defaultValue).ToString(null, CultureInfo.InvariantCulture)
239  : Invariant($"{defaultValue}");
240 
241  Log.Trace(Invariant($"Config.GetValue(): {key} - Using default value: {defaultValueString}"));
242  return defaultValue;
243  }
244 
245  var type = typeof(T);
246  string value;
247  try
248  {
249  value = token.Value<string>();
250  }
251  catch (Exception)
252  {
253  value = token.ToString();
254  }
255 
256  if (type.IsEnum)
257  {
258  return (T) Enum.Parse(type, value, true);
259  }
260 
261  if (typeof(IConvertible).IsAssignableFrom(type))
262  {
263  return (T) Convert.ChangeType(value, type, CultureInfo.InvariantCulture);
264  }
265 
266  // try and find a static parse method
267  try
268  {
269  var parse = type.GetMethod("Parse", new[]{typeof(string)});
270  if (parse != null)
271  {
272  var result = parse.Invoke(null, new object[] {value});
273  return (T) result;
274  }
275  }
276  catch (Exception err)
277  {
278  Log.Trace(Invariant($"Config.GetValue<{typeof(T).Name}>({key},{defaultValue}): Failed to parse: {value}. Using default value."));
279  Log.Error(err);
280  return defaultValue;
281  }
282 
283  try
284  {
285  return JsonConvert.DeserializeObject<T>(value);
286  }
287  catch (Exception err)
288  {
289  Log.Trace(Invariant($"Config.GetValue<{typeof(T).Name}>({key},{defaultValue}): Failed to JSON deserialize: {value}. Using default value."));
290  Log.Error(err);
291  return defaultValue;
292  }
293  }
294 
295  /// <summary>
296  /// Tries to find the specified key and parse it as a T, using
297  /// default(T) if unable to locate the key or unable to parse it
298  /// </summary>
299  /// <typeparam name="T">The desired output type</typeparam>
300  /// <param name="key">The configuration key</param>
301  /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else default(T).</param>
302  /// <returns>True on successful parse or if they key is not found. False only when key is found but fails to parse.</returns>
303  public static bool TryGetValue<T>(string key, out T value)
304  {
305  return TryGetValue(key, default(T), out value);
306  }
307 
308  /// <summary>
309  /// Tries to find the specified key and parse it as a T, using
310  /// defaultValue if unable to locate the key or unable to parse it
311  /// </summary>
312  /// <typeparam name="T">The desired output type</typeparam>
313  /// <param name="key">The configuration key</param>
314  /// <param name="defaultValue">The default value to use on key not found or unsuccessful parse</param>
315  /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else defaultValue.</param>
316  /// <returns>True on successful parse or if they key is not found and using defaultValue. False only when key is found but fails to parse.</returns>
317  public static bool TryGetValue<T>(string key, T defaultValue, out T value)
318  {
319  try
320  {
321  value = GetValue(key, defaultValue);
322  return true;
323  }
324  catch
325  {
326  value = defaultValue;
327  return false;
328  }
329  }
330 
331  /// <summary>
332  /// Write the contents of the serialized configuration back to the disk.
333  /// </summary>
334  public static void Write(string targetPath = null)
335  {
336  if (!Settings.IsValueCreated) return;
337  var serialized = JsonConvert.SerializeObject(Settings.Value, Formatting.Indented);
338 
339  var taget = ConfigurationFileName;
340  if (!string.IsNullOrEmpty(targetPath))
341  {
342  taget = Path.Combine(targetPath, ConfigurationFileName);
343  }
344  File.WriteAllText(taget, serialized);
345  }
346 
347  /// <summary>
348  /// Flattens the jobject with respect to the selected environment and then
349  /// removes the 'environments' node
350  /// </summary>
351  /// <param name="overrideEnvironment">The environment to use</param>
352  /// <returns>The flattened JObject</returns>
353  public static JObject Flatten(string overrideEnvironment)
354  {
355  return Flatten(Settings.Value, overrideEnvironment);
356  }
357 
358  /// <summary>
359  /// Flattens the jobject with respect to the selected environment and then
360  /// removes the 'environments' node
361  /// </summary>
362  /// <param name="config">The configuration represented as a JObject</param>
363  /// <param name="overrideEnvironment">The environment to use</param>
364  /// <returns>The flattened JObject</returns>
365  public static JObject Flatten(JObject config, string overrideEnvironment)
366  {
367  var clone = (JObject)config.DeepClone();
368 
369  // remove the environment declaration
370  var environmentProperty = clone.Property("environment");
371  if (environmentProperty != null) environmentProperty.Remove();
372 
373  if (!string.IsNullOrEmpty(overrideEnvironment))
374  {
375  var environmentSections = overrideEnvironment.Split('.');
376 
377  for (int i = 0; i < environmentSections.Length; i++)
378  {
379  var env = string.Join(".environments.", environmentSections.Where((x, j) => j <= i));
380 
381  var environments = config["environments"];
382  if (!(environments is JObject)) continue;
383 
384  var settings = ((JObject) environments).SelectToken(env);
385  if (settings == null) continue;
386 
387  // copy values for the selected environment to the root
388  foreach (var token in settings)
389  {
390  var path = Path.GetExtension(token.Path);
391  var dot = path.IndexOf(".", StringComparison.InvariantCulture);
392  if (dot != -1) path = path.Substring(dot + 1);
393 
394  // remove if already exists on clone
395  var jProperty = clone.Property(path);
396  if (jProperty != null) jProperty.Remove();
397 
398  var value = (token is JProperty ? ((JProperty) token).Value : token).ToString();
399  clone.Add(path, value);
400  }
401  }
402  }
403 
404  // remove all environments
405  var environmentsProperty = clone.Property("environments");
406  environmentsProperty?.Remove();
407 
408  return clone;
409  }
410 
411  private static JToken GetToken(JToken settings, string key)
412  {
413  return GetToken(settings, key, settings.SelectToken(key));
414  }
415 
416  private static JToken GetToken(JToken settings, string key, JToken current)
417  {
418  var environmentSetting = settings.SelectToken("environment");
419  if (environmentSetting != null)
420  {
421  var environmentSettingValue = environmentSetting.Value<string>();
422  if (!string.IsNullOrWhiteSpace(environmentSettingValue))
423  {
424  var environment = settings.SelectToken("environments." + environmentSettingValue);
425  if (environment != null)
426  {
427  var setting = environment.SelectToken(key);
428  if (setting != null)
429  {
430  current = setting;
431  }
432  // allows nesting of environments, live.tradier, live.interactive, ect...
433  return GetToken(environment, key, current);
434  }
435  }
436  }
437  if (current == null)
438  {
439  return settings.SelectToken(key);
440  }
441  return current;
442  }
443  }
444 }