Генератор аниме-фажества

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Data;
using Data.Enum;
using Data.Model;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Newtonsoft.Json;

namespace DanbooruDownloader.HostedServices
{
    public class FindWaifuService : IHostedService
    {
        private const double MIN_CHAR_FRANCHISE_INTERSECT_FOR_AFFILIATION = 0.85d;

        /// <summary>
        /// Фильтр фажества. Минимальный процент
        /// </summary>
        private const double MIN_COUNT_PERCENT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING = 0.28;

        private const double MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING = 10;
        private const double MAX_FAG_POINTS_PER_CHARACTER = 10;

        private const double FAG_POINTS_LOG10_MULTIPLIER = 0.15;

        private const int MIN_CHAR_COUNT_TO_CREATE_STAT = 10;
        private const int MAX_CHARACTERS_PER_IMAGE_FOR_FAGGING_COUNT = 2;

        private readonly NFContextFactory NFContextFactory;

        // ReSharper disable once NotAccessedField.Local
        private readonly IConfiguration Configuration;
        private readonly CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
        private CancellationToken CancellationToken => CancellationTokenSource.Token;

        private Task MainTaskItself;
        private Task Count123MeanTaskItself;

        public FindWaifuService(
            NFContextFactory nfContextFactory,
            IConfiguration configuration
        )
        {
            NFContextFactory = nfContextFactory;
            Configuration = configuration;
        }

        public Task StartAsync(CancellationToken cancellationToken)
        {
            MainTaskItself = Task.Run(SearchForWaifu, cancellationToken);
            Count123MeanTaskItself = Task.CompletedTask; // Task.Run(CalculateMeanCount123, cancellationToken);

            return Task.CompletedTask;
        }

        public async Task StopAsync(CancellationToken cancellationToken)
        {
            CancellationTokenSource.Cancel();
            await MainTaskItself;
            await Count123MeanTaskItself;
        }

        #region SearchForWaifu

        private DanbooruTag[] tags;
        private Dictionary<int, DanbooruTag> tagDictionary;
        private Dictionary<int, int> countsByTagId;
        private ICollection<int> tagArtistIds;
        private ICollection<int> tagCharacterIds;
        private ICollection<DanbooruTagIntersect> intersects;
        private IDictionary<int, int> parentImages;

        private int GetRealImageId(int imageId)
        {
            return parentImages.ContainsKey(imageId) ? parentImages[imageId] : imageId;
        }

        private IDictionary<int, int> characterToCopyright;
        private int tagOriginalId;

        private async Task SearchForWaifu_GenerateCopyright(NFContext context)
        {
            var tagCopyrightIds = tags
                .Where(x => x.type == DanbooruTagType.Copyright)
                .Where(x => x.id != tagOriginalId)
                .Select(t => t.id)
                .ToHashSet();
            Console.WriteLine(
                "{0}\ttagCopyrightIds.Length = {1,7}",
                DateTimeOffset.UtcNow,
                tagCopyrightIds.Count()
            );
            var i1 = await context
                .DanbooruTagIntersect
                .Where(x => tagCopyrightIds.Contains(x.tag_id))
                .ToArrayAsync();
            Console.WriteLine(
                "{0}\tDb: i1.Length = {1,7}",
                DateTimeOffset.UtcNow,
                i1.Count()
            );

            i1 = i1
                .Select(t => (art_id: GetRealImageId(t.art_id), t.tag_id))
                .Distinct()
                .Select(t => new DanbooruTagIntersect() {art_id = t.art_id, tag_id = t.tag_id})
                .ToArray();
            Console.WriteLine(
                "{0}\tParented: i1.Length = {1,7}",
                DateTimeOffset.UtcNow,
                i1.Count()
            );

            var franchisesByArtId = i1
                .GroupBy(t => t.art_id)
                .Select(t => (art_id: t.Key, copyrights: t.Select(t1 => t1.tag_id).Distinct().ToArray()))
                .Where(x => x.copyrights.Length == 1)
                .ToDictionary(
                    t => t.art_id,
                    t => t.copyrights.First()
                );
            var charCounts1 = intersects
                .Where(x => tagCharacterIds.Contains(x.tag_id))
                .GroupBy(t => t.tag_id)
                .ToDictionary(
                    t => t.Key,
                    t => t.Count()
                );
            var charCounts2 = intersects
                .Where(x => franchisesByArtId.ContainsKey(x.art_id) && tagCharacterIds.Contains(x.tag_id))
                .GroupBy(t => t.tag_id)
                .ToDictionary(
                    t => t.Key,
                    t => t.Count()
                );
            Console.WriteLine(
                "{0}\tfranchisesByArtId.Length = {1,7}" +
                "\n\t\t\t\tcharCounts1.Length = {2,7}" +
                "\n\t\t\t\tcharCounts2.Length = {3,7}",
                DateTimeOffset.UtcNow,
                franchisesByArtId.Count(),
                charCounts1.Count(),
                charCounts2.Count()
            );

            characterToCopyright = intersects
                .Where(x => tagCharacterIds.Contains(x.tag_id) && charCounts1.ContainsKey(x.tag_id))
                .Select(t =>
                {
                    // ReSharper disable once ConvertToLambdaExpression
                    return (charId: t.tag_id,
                        copyright: franchisesByArtId.ContainsKey(t.art_id) ? franchisesByArtId[t.art_id] : -1);
                })
                .Where(x => x.copyright != -1)
                .GroupBy(t => t.charId)
                .Select(t =>
                {
                    var charId = t.Key;
                    var minCount = charCounts1[charId] * MIN_CHAR_FRANCHISE_INTERSECT_FOR_AFFILIATION;

                    var (copyright, count) = t
                        .GroupBy(t1 => t1.copyright)
                        .Select(t1 => (t1.Key, count: t1.Count()))
                        .OrderByDescending(t1 => t1.count)
                        .First();
                    // ReSharper disable once ConvertIfStatementToReturnStatement
                    if (count >= minCount)
                    {
                        return (charId, copyright: copyright);
                    }

                    if (!charCounts2.ContainsKey(charId))
                    {
                        return (charId, copyright: -1);
                    }

                    minCount = charCounts2[charId] * MIN_CHAR_FRANCHISE_INTERSECT_FOR_AFFILIATION;
                    // ReSharper disable once ConvertIfStatementToReturnStatement
                    if (count >= minCount)
                    {
                        return (charId, copyright: copyright);
                    }

                    return (charId, copyright: -1);
                })
                .Where(x => x.copyright != -1)
                .OrderBy(t => t.charId)
                .ToDictionary(
                    t => t.charId,
                    t => t.copyright
                );
            Console.WriteLine(
                "{0}\tcharacterToCopyright.Length = {1,7}",
                DateTimeOffset.UtcNow,
                characterToCopyright.Count()
            );
        }

        private async Task SearchForWaifu_Step1()
        {
            Console.WriteLine(
                "{0}\tSearchForWaifu_Step1",
                DateTimeOffset.UtcNow
            );
            await using var context = NFContextFactory.CreateContext();
            Console.WriteLine(
                "{0}\tDB connected",
                DateTimeOffset.UtcNow
            );
            parentImages = await context
                .DanbooruImage
                .Where(x => x.parent_id != null)
                .ToDictionaryAsync(
                    t => t.id,
                    t => t.parent_id!.Value
                );
            Console.WriteLine(
                "{0}\tparentImages.Length = {1,7}",
                DateTimeOffset.UtcNow,
                parentImages.Count()
            );

            tags = await context
                .DanbooruTag
                .ToArrayAsync();
            tagDictionary = tags.ToDictionary(t => t.id, t => t);
            Console.WriteLine(
                "{0}\ttags.Length = {1,7}",
                DateTimeOffset.UtcNow,
                tags.Count()
            );
            tagOriginalId = tags.First(x => x.tag == "original").id;

            tagArtistIds = tags
                .Where(x => x.type == DanbooruTagType.Artist)
                .Select(t => t.id)
                .ToHashSet();
            tagCharacterIds = tags
                .Where(x => x.type == DanbooruTagType.Character)
                .Select(t => t.id)
                .ToHashSet();
            var artistCharacterIds = tagArtistIds.Concat(tagCharacterIds).ToArray();
            Console.WriteLine(
                "{0}\ttagArtistIds.Length = {1,7}" +
                "\n\t\t\t\ttagCharacterIds = {2,7}" +
                "\n\t\t\t\tOriginal tag = {3,7}",
                DateTimeOffset.UtcNow,
                tagArtistIds.Count(),
                tagCharacterIds.Count(),
                tagOriginalId
            );

            var sw = Stopwatch.StartNew();
            intersects = await context
                .DanbooruTagIntersect
                .Where(x => artistCharacterIds.Contains(x.tag_id))
                .ToArrayAsync();
            sw.Stop();
            var intersectsRawCount = intersects.Count;
            Console.WriteLine(
                "{0}\tDb: intersects.Length = {1,7}",
                DateTimeOffset.UtcNow,
                intersects.Count()
            );
            intersects = intersects
                .Select(t => (art_id: GetRealImageId(t.art_id), t.tag_id))
                .Distinct()
                .Select(t => new DanbooruTagIntersect()
                {
                    art_id = t.art_id,
                    tag_id = t.tag_id,
                })
                .ToArray();
            Console.WriteLine(
                "{0}\tParented: intersects.Length = {1,7}",
                DateTimeOffset.UtcNow,
                intersects.Count()
            );
            await SearchForWaifu_GenerateCopyright(context);

            {
                var artId1 = intersects
                    .Where(x => tagArtistIds.Contains(x.tag_id))
                    .Select(t => t.art_id)
                    .ToHashSet();
                artId1 = intersects
                    .Where(x => tagCharacterIds.Contains(x.tag_id))
                    .Select(t => t.art_id)
                    .Where(x => artId1.Contains(x))
                    .ToHashSet();
                Console.WriteLine(
                    "{0}\tClear artId1 with characters and artists = {1,7}",
                    DateTimeOffset.UtcNow,
                    artId1.Count()
                );

                intersects = intersects
                    .Where(x => artId1.Contains(x.art_id))
                    .ToArray();
            }

            ClearIntersectsFromMultipleChars();
            await ClearIntersectsFromBadTags(context);

            countsByTagId = intersects
                .GroupBy(t => t.tag_id)
                .ToDictionary(
                    t => t.Key,
                    t => t.Count()
                );

            Console.WriteLine(
                "{0}\tGet intersects: {6}" +
                "\n\t\t\t\tArtists count = {1,6}" +
                "\n\t\t\t\tCharacter count = {2,6}" +
                "\n\t\t\t\tIntersects {3,7} ({4,7} {5:P1})" +
                "\n\t\t\t\tcountsByTagId.Length = {7,7}",
                DateTimeOffset.UtcNow,
                tagArtistIds.Count,
                tagCharacterIds.Count,
                intersects.Count,
                intersectsRawCount,
                intersects.Count * 1d / intersectsRawCount,
                sw.Elapsed,
                countsByTagId.Count()
            );
        }

        /// <summary>
        /// Delete game_cg & Delete official art
        /// </summary>
        /// <param name="context"></param>
        private async Task ClearIntersectsFromBadTags(NFContext context)
        {
            var gameCG = tags.First(x => x.tag == "game_cg").id;
            var officialArt = tags.First(x => x.tag == "official_art").id;
            var a = new[] {gameCG, officialArt};

            ICollection<int> badArtIds = await context
                .DanbooruTagIntersect
                .Where(x => a.Contains(x.tag_id))
                .Select(t => t.art_id)
                .Distinct()
                .ToArrayAsync();
            badArtIds = badArtIds.ToHashSet();
            intersects = intersects
                .Where(x => !badArtIds.Contains(x.art_id))
                .ToArray();

            Console.WriteLine(
                "{0}\tClearIntersectsFromBadTags: badArtIds.Length = {1,7}" +
                "\n\t\t\t\tintersects.Length = {2,7}",
                DateTimeOffset.UtcNow,
                badArtIds.Count(),
                intersects.Count()
            );
        }

        /// <summary>
        /// Delete multi-characters (4+) arts
        /// </summary>
        private void ClearIntersectsFromMultipleChars()
        {
            var badArtIds = intersects
                .Where(x => tagCharacterIds.Contains(x.tag_id))
                .GroupBy(t => t.art_id)
                .Select(t => (art_id: t.Key, count: t.Count()))
                .Where(x => x.count > MAX_CHARACTERS_PER_IMAGE_FOR_FAGGING_COUNT)
                .Select(t => t.art_id)
                .ToHashSet();
            intersects = intersects
                .Where(x => !badArtIds.Contains(x.art_id))
                .ToArray();

            Console.WriteLine(
                "{0}\tClearIntersectsFromMultipleChars: badArtIds.Length = {1,7}" +
                "\n\t\t\t\tintersects.Length = {2,7}",
                DateTimeOffset.UtcNow,
                badArtIds.Count(),
                intersects.Count()
            );
        }

        private async Task SearchForWaifu()
        {
            Console.WriteLine(
                "{0}\tSearchForWaifu",
                DateTimeOffset.UtcNow
            );
            await SearchForWaifu_Step1();
            var byArts = intersects
                .GroupBy(t => t.art_id)
                .Select(t =>
                {
                    var a = t.Select(t1 => t1.tag_id).ToArray();
                    var artists = a.Where(tagArtistIds.Contains).ToArray();
                    var characters = a.Where(tagCharacterIds.Contains).ToArray();

                    return (artId: t.Key, artists, characters);
                })
                .Where(x => x.artists.Any() && x.characters.Any())
                .ToDictionary(
                    t => t.artId,
                    t => t
                );
            Console.WriteLine(
                "{0}\tbyArts.Length = {1,7}",
                DateTimeOffset.UtcNow,
                byArts.Count()
            );

            var semiRawData = byArts
                .Values
                .SelectMany(t =>
                {
                    var (_, artists, characters) = t;
                    return artists
                        .SelectMany(t1 => characters.Select(t2 => (artist: t1, character: t2)));
                })
                .GroupBy(t => t)
                .Select(t => (key: t.Key, count: t.Count()))
                .Select(t => (key: t.key, count: t.count, countP: t.count * 1d / countsByTagId[t.key.artist]))
                .ToArray();
            var countDictionary = semiRawData
                .Where(x => x.count > MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING)
                .OrderByDescending(t => t.countP + Math.Log10(t.count) * FAG_POINTS_LOG10_MULTIPLIER)
                .ToDictionary(
                    t => t.key,
                    t => t.count
                );
            Console.WriteLine(
                "{0}\tsemiRawData.Length = {1,7}" +
                "\n\t\t\t\tcountDictionary = {2,7}",
                DateTimeOffset.UtcNow,
                semiRawData.Count(),
                countDictionary.Count()
            );

            {
                // ReSharper disable once NotAccessedVariable
                var realCount = 0;
                var lines = new List<string>();
                foreach (var ((artist, character), count) in countDictionary /*.TakeWhile(_ => realCount < 5000)*/)
                {
                    var artistCount = countsByTagId[artist];
                    var characterCount = countsByTagId[character];
                    if (characterCount - count < MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING)
                    {
                        continue;
                    }

                    if (count < artistCount * 0.1)
                    {
                        continue;
                    }

                    realCount++;

                    lines.Add(string.Format(
                        "+------T------+\tartist = {0,-30}\tcharacter = {1}",
                        tagDictionary[artist].tag,
                        tagDictionary[character].tag
                    ));
                    lines.Add(string.Format(
                        "|{0,6}|{1,6}|",
                        0,
                        artistCount - count
                    ));
                    lines.Add("+------+------+");
                    lines.Add(string.Format(
                        "|{0,6}|{1,6}|",
                        characterCount - count,
                        count
                    ));
                    lines.Add("+------+------+");
                    lines.Add(string.Format(
                        "Arts by the artist to this character: {0,-6:P1}\n" +
                        "Arts with the char by this artist:    {1,-6:P1}",
                        count * 1d / artistCount,
                        count * 1d / characterCount
                    ));

                    lines.Add("=============================================");
                }

                await File.WriteAllLinesAsync(@"waifu-list.txt", lines, CancellationToken);

                Console.WriteLine(
                    "{0}\twaifu-list.txt done",
                    DateTimeOffset.UtcNow
                );
            }

            {
                // 1й, 2й, 3й персонаж для каждого художника
                var count123 = semiRawData
                    .GroupBy(t => t.key.artist)
                    .Select(t =>
                    {
                        var sum = t.Sum(t1 => t1.count);
                        if (sum < MIN_CHAR_COUNT_TO_CREATE_STAT)
                        {
                            return Array.Empty<double>();
                        }

                        var r = 1d / sum;
                        var a = t
                            .OrderByDescending(t1 => t1.count)
                            .Take(3)
                            .Select(t1 => t1.count * r)
                            .ToArray();

                        return a.Length switch
                        {
                            3 => a,
                            1 => a.Concat(new[] {0d, 0d}).ToArray(),
                            2 => a.Concat(new[] {0d}).ToArray(),
                            _ => throw new ArgumentOutOfRangeException()
                        };
                    })
                    .Where(x => x.Any())
                    .ToArray();
                Console.WriteLine(
                    "{0}\tcount123.Length = {1,7}",
                    DateTimeOffset.UtcNow,
                    count123.Count()
                );

                // TODO 1й, 2й, 3й персонаж для каждого художника
                var textInput = JsonConvert.SerializeObject(count123);
                await File.WriteAllTextAsync(@"count123-input.json", textInput);
            }

            {
                var rawCharFag = semiRawData
                    .Where(x => (x.count >= MIN_COUNT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING) &&
                                (x.countP >= MIN_COUNT_PERCENT_BY_ARTIST_TO_CHARACTER_FOR_FAGGING))
                    .Select(t => (fagPoint: t.countP + Math.Log10(t.count) * FAG_POINTS_LOG10_MULTIPLIER,
                        t.key.character))
                    .ToArray();

                var fagCharacters = rawCharFag
                    .GroupBy(t => t.character)
                    .Select(t => (character: t.Key, sum: t.Sum(t1 => t1.fagPoint), count: t.Count()))
                    .OrderByDescending(t => t.sum)
                    .ToArray();
                Console.WriteLine(
                    "{0}\trawCharFag.Length = {1,7}" +
                    "\n\t\t\t\tfagCharacters = {2,7}",
                    DateTimeOffset.UtcNow,
                    rawCharFag.Count(),
                    fagCharacters.Count()
                );

                var lines = new List<string>();
                foreach (var (charId, sum, count) in fagCharacters)
                {
                    var character = tagDictionary[charId];
                    lines.Add(string.Format("{0}\t{1:F2}\t{2}", character.tag, sum, count));
                }

                await File.WriteAllLinesAsync(@"character-fag-list.txt", lines, CancellationToken);
                Console.WriteLine(
                    "{0}\tfag-list.txt done",
                    DateTimeOffset.UtcNow
                );

                //
                var franchiseFagDic = fagCharacters
                    .Select(t =>
                    {
                        var (charId, fagPoint, _) = t;
                        if (!characterToCopyright.ContainsKey(charId))
                        {
                            return (copyright: -1, fagPoint: 0d);
                        }

                        var copyright = characterToCopyright[charId];

                        return (copyright, fagPoint);
                    })
                    .Where(x => x.copyright != -1)
                    .GroupBy(t => t.copyright)
                    .ToDictionary(
                        t => t.Key,
                        t => (
                            tag: tagDictionary[t.Key],
                            rawSum: t.Sum(t1 => t1.fagPoint),
                            limitedSum: t.Sum(t1 => Math.Min(MAX_FAG_POINTS_PER_CHARACTER, t1.fagPoint)),
                            charCount: t.Count()
                        )
                    );

                var lines1 = franchiseFagDic
                    .OrderByDescending(t => t.Value.rawSum)
                    .Take(100)
                    .Select(t =>
                    {
                        var fagPoint = t.Value.rawSum;

                        return string.Format("{0}\t{1:F2}\t{2}", t.Value.tag.tag, fagPoint, t.Value.charCount);
                    })
                    .ToArray();
                await File.WriteAllLinesAsync(@"franchise-fag-list1.txt", lines1, CancellationToken);

                var lines2 = franchiseFagDic
                    .OrderByDescending(t => t.Value.limitedSum)
                    .Take(100)
                    .Select(t =>
                    {
                        var fagPoint = t.Value.limitedSum;

                        return string.Format("{0}\t{1:F2}\t{2}", t.Value.tag.tag, fagPoint, t.Value.charCount);
                    })
                    .ToArray();
                await File.WriteAllLinesAsync(@"franchise-fag-list2.txt", lines2, CancellationToken);

                Console.WriteLine(
                    "{0}\tfranchise-fag-list [both].txt done",
                    DateTimeOffset.UtcNow
                );
            }
        }

        #endregion

        #region CalculateMeanCount123

        private async Task CalculateMeanCount123()
        {
            List<(double[] v, int count)[]> count123Chunks;
            const int CHUNK_SIZE = 1000;
            const int MIN_CHUNK_SIZE = 500;
            {
                var text = await File.ReadAllTextAsync(@"count123-input.json");
                var rawCount123 = JsonConvert.DeserializeObject<double[][]>(text);
                var rnd = new Random();

                var lists = new List<(double[] v, int count)[]>();
                var currentList = new List<(double[] v, int count)>();
                foreach (var values in rawCount123.OrderBy(_ => rnd.NextDouble()))
                {
                    currentList.Add((values, 1));
                    if (currentList.Count() >= CHUNK_SIZE)
                    {
                        lists.Add(currentList
                            .OrderByDescending(t => t.v[0])
                            .ThenByDescending(t => t.v[1])
                            .ToArray());
                        currentList = new List<(double[] v, int count)>();
                    }
                }

                if (currentList.Any())
                {
                    lists.Add(currentList
                        .OrderByDescending(t => t.v[0])
                        .ThenByDescending(t => t.v[1])
                        .ToArray());
                }

                count123Chunks = lists.ToList();
            }

            var fullCount = count123Chunks.Sum(t => t.Sum(t1 => t1.count));

            (double[] v, int count)[] count123 = count123Chunks.First();
            count123Chunks.RemoveAt(0);
            var sw = Stopwatch.StartNew();

            while (count123.Length > 2)
            {
                var len = count123.Length;
                Console.WriteLine(
                    "\ncount123.Length = {0} + {1,4} * {2,5}",
                    count123.Length,
                    count123Chunks.Count(),
                    CHUNK_SIZE
                );

                var calculatedPairs = Enumerable
                    .Range(1, len - 1)
                    .SelectMany(index1 =>
                    {
                        return Enumerable
                            .Range(0, index1)
                            .Select(index2 => (index1, index2));
                    })
                    .AsParallel()
                    .Select(t =>
                    {
                        var (index1, index2) = t;
                        // ReSharper disable once AccessToModifiedClosure
                        var item1 = count123[index1];
                        // ReSharper disable once AccessToModifiedClosure
                        var item2 = count123[index2];

                        var sqrt = 0d;
                        for (var i = 0; i < 3; i++)
                        {
                            sqrt += Math.Pow(item1.v[i] - item2.v[i], 2);
                        }

                        sqrt = Math.Sqrt(sqrt / 3d);

                        return (index1: index1, index2: index2, sqrt);
                    })
                    .OrderBy(t => t.sqrt)
                    .ToArray();

                var bestPair = calculatedPairs.First();
                Console.WriteLine(bestPair);
                if (bestPair.sqrt > 0.05)
                {
                    break;
                }

                {
                    var item1 = count123[bestPair.index1];
                    var item2 = count123[bestPair.index2];

                    var r1 = item1.count * 1d / (item1.count + item2.count);
                    var r2 = 1 - r1;

                    var newItem = (
                        v: Enumerable.Range(0, 3).Select(i => item1.v[i] * r1 + item2.v[i] * r2).ToArray(),
                        count: item1.count + item2.count
                    );
                    Console.WriteLine("\tnew item\t{1,-5}\t{0}",
                        newItem.v.Select(t => t.ToString("F3")).Aggregate((a, b) => a + ", " + b),
                        newItem.count);

                    count123 = count123
                        .Select((t, index) => (t, index))
                        .Where(x => (x.index != bestPair.index1) && (x.index != bestPair.index2))
                        .Select(t => t.t)
                        .Concat(new[] {newItem})
                        .OrderByDescending(t => t.count)
                        .ToArray();
                }

                if ((count123.Length <= MIN_CHUNK_SIZE) && (count123Chunks.Any()))
                {
                    var elapsed = sw.Elapsed;
                    var leftCount = count123.Length + count123Chunks.Count * CHUNK_SIZE;
                    var p = 1d - leftCount * 1d / fullCount;
                    Console.WriteLine(
                        "--------------------------\t{0,-7:P1} (elapsed: {1})\tETA: {2}",
                        p,
                        elapsed,
                        (1 - p) / p * elapsed
                    );

                    var r = 1d / count123.Sum(t => t.count);
                    for (int i = 0; i < Math.Min(10, count123.Length); i++)
                    {
                        var (v, count) = count123[i];
                        Console.WriteLine("\t{1,-5}\t{0}\t{2,6:P1}",
                            v.Select(t => t.ToString("F3")).Aggregate((a, b) => a + ", " + b),
                            count,
                            count * r
                        );
                    }

                    count123 = count123
                        .Concat(count123Chunks.First())
                        .ToArray();
                    count123Chunks.RemoveAt(0);
                }
            }

            Console.WriteLine("==================================================");
            Console.WriteLine("==================================================");
            Console.WriteLine("==================================================");
            var rFullCount = 1d / fullCount;
            for (int i = 0; i < Math.Min(20, count123.Length); i++)
            {
                var (v, count) = count123[i];
                Console.WriteLine("\t{1,-5}\t{0}\t{2,6:P1}",
                    v.Select(t => t.ToString("F3")).Aggregate((a, b) => a + ", " + b),
                    count,
                    count * rFullCount
                );
            }
        }

        #endregion
    }
}