AlgoSeekFuturesProgram.cs

/*
 * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
 * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/
 
using System;
using QuantConnect.Logging;
using System.Diagnostics;
using System.Globalization;
using QuantConnect.Configuration;
using System.Linq;
using System.IO;
 
namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter
{
    /// <summary>
    /// AlgoSeek Options Converter: Convert raw OPRA channel files into QuantConnect Options Data Format.
    /// </summary>
    public static class AlgoSeekFuturesProgram
    {
        public static void AlgoSeekFuturesConverter(string date)
        {
            // There are practical file limits we need to override for this to work.
            // By default programs are only allowed 1024 files open; for futures parsing we need 100k
            Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "disabled");
            Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("log.txt") });
 
            // Directory for the data, output and processed cache:
            var remoteDirectory = Config.Get("futures-remote-directory").Replace("{0}", date);
            var sourceDirectory = Config.Get("futures-source-directory").Replace("{0}", date);
            var dataDirectory = Globals.DataFolder;
            var resolutions = Config.Get("resolutions");
            var cleanSourceDirectory = Config.GetBool("clean-source-directory", false);
 
            Log.Trace("CONFIGURATION:");
            Log.Trace("Processor Count: " + Environment.ProcessorCount);
            Log.Trace("Remote Directory: " + remoteDirectory);
            Log.Trace("Source Directory: " + sourceDirectory);
            Log.Trace("Destination Directory: " + dataDirectory);
 
            // Date for the option bz files.
            var referenceDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
 
            Log.Trace("DateTime: " + referenceDate.Date.ToStringInvariant());
 
            // checking if remote folder exists
            if(!Directory.Exists(remoteDirectory))
            {
                Log.Error("Remote Directory doesn't exist: " + remoteDirectory);
                return;
            }
 
            // prepare tick types
            var resolutionList = new[] { Resolution.Minute };
 
            if (!string.IsNullOrEmpty(resolutions))
            {
                var names = resolutions.Split(new[] { ';' });
                resolutionList =
                    names
                    .Where(x => !string.IsNullOrEmpty(x))
                    .Select(name => (Resolution)Enum.Parse(typeof(Resolution), name, true)).ToArray();
            }
 
            Log.Trace("Resolutions: " + string.Join(";", resolutionList.Select(x => x.ToString()).ToArray()));
 
            // Convert the date:
            var timer = Stopwatch.StartNew();
            var converter = new AlgoSeekFuturesConverter(resolutionList.ToList() , referenceDate, remoteDirectory, sourceDirectory, dataDirectory);
            converter.Convert();
            Log.Trace($"AlgoSeekFuturesConverter.Main(): {referenceDate.ToStringInvariant()} Conversion finished in time: {timer.Elapsed.ToStringInvariant(null)}");
 
            // Compress the memory cache to zips.
            timer.Restart();
            converter.Package(referenceDate);
            Log.Trace($"AlgoSeekFuturesConverter.Main(): {referenceDate.ToStringInvariant()} Compression finished in time: {timer.Elapsed.ToStringInvariant(null)}");
 
            if (cleanSourceDirectory)
            {
                Log.Trace($"AlgoSeekFuturesConverter.Main(): Cleaning source directory: {sourceDirectory}");
 
                try
                {
                    Directory.Delete(sourceDirectory, true);
                }
                catch(Exception err)
                {
                    Log.Trace($"AlgoSeekFuturesConverter.Main(): Error while cleaning source directory {err.Message}");
                }
            }
        }
    }
}