The example illustrates a "clock-time" benchmark for assessing detection speed.Using a YAML formatted evidence file - "20000 Evidence Records.yml" - supplied with the distribution or can be obtained from the data repository on Github.
It's important to understand the trade-offs between performance, memory usage and accuracy, that the 51Degrees pipeline configuration makes available, and this example shows a range of different configurations to illustrate the difference in performance.
Requesting properties from a single component reduces detection time compared with requesting properties from multiple components. If you don't specify any properties to detect, then all properties are detected.
This example requires a local data file. The free 'Lite' data file can be acquired by
pulling the git submodules under this repository (run `git submodule update --recursive`)
or from the device-detection-data
GitHub repository.
The Lite data file is only used for illustration, and has limited accuracy and capabilities.
Find out about the more capable data files that are available on our
pricing page
using FiftyOne.Pipeline.Core.FlowElements;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
{
public class Program
{
private class BenchmarkResult
{
public long Count { get; set; }
public Stopwatch Timer { get; } = new Stopwatch();
public long MobileCount { get; set; }
public long NotMobileCount { get; set; }
}
private static readonly PerformanceConfiguration[] _configs = new PerformanceConfiguration[]
{
new PerformanceConfiguration(true, PerformanceProfiles.MaxPerformance, false, true, false),
new PerformanceConfiguration(true, PerformanceProfiles.MaxPerformance, true, true, false),
new PerformanceConfiguration(false, PerformanceProfiles.LowMemory, false, true, false),
new PerformanceConfiguration(false, PerformanceProfiles.LowMemory, true, true, false)
};
private const ushort DEFAULT_THREAD_COUNT = 4;
public class Example : ExampleBase
{
private IPipeline _pipeline;
public Example(IPipeline pipeline)
{
_pipeline = pipeline;
}
private void Run(
TextReader evidenceReader,
TextWriter output,
int threadCount)
{
output.WriteLine("Warming up");
var warmup = Benchmark(evidence, threadCount);
var warmupTime = warmup.Sum(r => r.Timer.ElapsedMilliseconds);
GC.Collect();
Task.Delay(500).Wait();
output.WriteLine("Running");
var execution = Benchmark(evidence, threadCount);
var executionTime = execution.Sum(r => r.Timer.ElapsedMilliseconds);
output.WriteLine($"Finished - Execution time was {executionTime} ms, " +
$"adjustment from warm-up {executionTime - warmupTime} ms");
Report(execution, threadCount, output);
}
private void Report(List<BenchmarkResult> results,
int threadCount,
TextWriter output)
{
var detections = results.Sum(r => r.Count);
var milliseconds = results.Sum(r => r.Timer.ElapsedMilliseconds);
var msPerDetection = (float)(milliseconds / threadCount) / detections;
var detectionsPerSecond = 1000 / msPerDetection;
output.WriteLine($"Overall: {detections} detections, Average millisecs per " +
$"detection: {msPerDetection}, Detections per second: {detectionsPerSecond}");
output.WriteLine($"Overall: Concurrent threads: {threadCount}");
}
private List<BenchmarkResult> Benchmark(
List<Dictionary<string, object>> allEvidence,
int threadCount)
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
List<BenchmarkResult> results = new List<BenchmarkResult>();
var processing = Parallel.ForEach(allEvidence,
new ParallelOptions()
{
MaxDegreeOfParallelism = threadCount,
CancellationToken = cancellationTokenSource.Token
},
() => new BenchmarkResult(),
(evidence, loopState, result) =>
{
result.Timer.Start();
using (var data = _pipeline.CreateFlowData())
{
data.AddEvidence(evidence).Process();
var device = data.Get<IDeviceData>();
result.Count++;
if(device.IsMobile.HasValue && device.IsMobile.Value)
{
result.MobileCount++;
}
else
{
result.NotMobileCount++;
}
}
result.Timer.Stop();
return result;
},
(result) =>
{
lock (results)
{
results.Add(result);
}
});
return results;
}
public static void Run(string dataFile, string evidenceFile,
PerformanceConfiguration config, TextWriter output, ushort threadCount)
{
using (var serviceProvider = new ServiceCollection()
.AddLogging(l => l.AddConsole())
.AddTransient<PipelineBuilder>()
.AddTransient<DeviceDetectionHashEngineBuilder>()
.AddSingleton((x) =>
{
var builder = x.GetRequiredService<DeviceDetectionHashEngineBuilder>()
.SetDataFileSystemWatcher(false)
.SetAutoUpdate(false)
.SetDataUpdateOnStartup(false)
.SetPerformanceProfile(config.Profile)
.SetUsePerformanceGraph(config.PerformanceGraph)
.SetUsePredictiveGraph(config.PredictiveGraph)
.SetConcurrency(threadCount);
if (config.AllProperties == false)
{
builder.SetProperty("IsMobile");
}
DeviceDetectionHashEngine engine = null;
if (config.LoadFromDisk)
{
engine = builder.Build(dataFile, false);
}
else
{
using (MemoryStream stream = new MemoryStream(File.ReadAllBytes(dataFile)))
{
engine = builder.Build(stream);
}
}
return engine;
})
.AddSingleton((x) => {
return x.GetRequiredService<PipelineBuilder>()
.AddFlowElement(x.GetRequiredService<DeviceDetectionHashEngine>())
.Build();
})
.AddTransient<Example>()
.BuildServiceProvider())
using (var evidenceReader = new StreamReader(File.OpenRead(evidenceFile)))
{
if (string.IsNullOrWhiteSpace(dataFile))
{
serviceProvider.GetRequiredService<ILogger<Program>>().LogError(
"Failed to find a device detection data file. Make sure the " +
"device-detection-data submodule has been updated by running " +
"`git submodule update --recursive`.");
}
else
{
ExampleUtils.CheckDataFile(
serviceProvider.GetRequiredService<IPipeline>(),
serviceProvider.GetRequiredService<ILogger<Program>>());
output.WriteLine($"Processing evidence from '{evidenceFile}'");
output.WriteLine($"Data loaded from '{(config.LoadFromDisk ? "disk" : "memory")}'");
output.WriteLine($"Benchmarking with profile '{config.Profile}', " +
$"AllProperties {config.AllProperties}, " +
$"PerformanceGraph {config.PerformanceGraph}, " +
$"PredictiveGraph {config.PredictiveGraph}");
serviceProvider.GetRequiredService<Example>().Run(evidenceReader, output, threadCount);
}
}
}
}
static void Main(string[] args)
{
var dataFile = args.Length > 0 ? args[0] :
ExampleUtils.FindFile(Constants.LITE_HASH_DATA_FILE_NAME);
var evidenceFile = args.Length > 1 ? args[1] :
ExampleUtils.FindFile(Constants.YAML_EVIDENCE_FILE_NAME);
foreach (var config in _configs)
{
Example.Run(dataFile, evidenceFile, config, Console.Out, DEFAULT_THREAD_COUNT);
}
}
}
}