Lean  $LEAN_TAG$
ProcessedDataProvider.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4 */
5 
6 using System.IO;
10 using System;
11 
13 {
14  /// <summary>
15  /// A data provider that will check the processed data folder first
16  /// </summary>
17  public class ProcessedDataProvider : IDataProvider, IDisposable
18  {
19  private readonly DefaultDataProvider _defaultDataProvider;
20  private readonly string _processedDataDirectory;
21 
22  /// <summary>
23  /// Ignored
24  /// </summary>
25  public event EventHandler<DataProviderNewDataRequestEventArgs> NewDataRequest;
26 
27  /// <summary>
28  /// Creates a new instance
29  /// </summary>
31  {
32  _defaultDataProvider = new();
33  _processedDataDirectory = Config.Get("processed-data-directory") ?? string.Empty;
34  Log.Trace($"ProcessedDataProvider(): processed data directory to use {_processedDataDirectory}, exists: {Directory.Exists(_processedDataDirectory)}");
35  }
36 
37  /// <summary>
38  /// Retrieves data from disc to be used in an algorithm
39  /// </summary>
40  /// <param name="key">A string representing where the data is stored</param>
41  /// <returns>A <see cref="Stream"/> of the data requested</returns>
42  public Stream Fetch(string key)
43  {
44  Stream result = null;
45 
46  // we will try the processed data folder first
47  if (_processedDataDirectory.Length != 0 && key.StartsWith(Globals.DataFolder, StringComparison.OrdinalIgnoreCase))
48  {
49  result = _defaultDataProvider.Fetch(Path.Combine(_processedDataDirectory, key.Remove(0, Globals.DataFolder.Length).TrimStart('/', '\\')));
50  if (result != null)
51  {
52  Log.Trace($"ProcessedDataProvider.Fetch({key}): fetched from processed data directory");
53  }
54  }
55 
56  // fall back to existing data folder path
57  return result ?? _defaultDataProvider.Fetch(key);
58  }
59 
60  /// <summary>
61  /// Disposes of resources
62  /// </summary>
63  public void Dispose()
64  {
65  Dispose(true);
66  GC.SuppressFinalize(this);
67  }
68 
69  /// <summary>
70  /// Disposes of the internal data provider
71  /// </summary>
72  protected virtual void Dispose(bool disposing)
73  {
74  if (disposing)
75  {
76  _defaultDataProvider.Dispose();
77  }
78  }
79  }
80 }