Skip to content

Commit 3618205

Browse files
jemejeme
authored
Merge pull request #5 from dotJEM/parser-changes
Parser changes
2 parents 036dea7 + c06ec79 commit 3618205

40 files changed

+1286
-237
lines changed

‎src/DotJEM.Json.Index2.Contexts.Test/DotJEM.Json.Index2.Contexts.Test.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
</PropertyGroup>
1313

1414
<ItemGroup>
15+
<PackageReference Include="Bogus" Version="35.5.1" />
1516
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
1617
<PackageReference Include="NUnit" Version="3.14.0" />
1718
</ItemGroup>

‎src/DotJEM.Json.Index2.Management.Test/DotJEM.Json.Index2.Management.Test.csproj

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="Bogus" Version="35.5.1" />
11+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
12+
<PackageReference Include="NUnit" Version="3.14.0" />
13+
</ItemGroup>
14+
15+
<ItemGroup>
16+
<ProjectReference Include="..\DotJEM.Json.Index2.Management\DotJEM.Json.Index2.Management.csproj" />
17+
</ItemGroup>
18+
19+
</Project>

‎src/DotJEM.Json.Index2.Management.Test/JsonIndexManagerTest.cs

+202
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
using System.Diagnostics;
2+
using System.Reactive.Linq;
3+
using System.Reflection.Metadata.Ecma335;
4+
using System.Runtime.CompilerServices;
5+
using Bogus;
6+
using DotJEM.AdvParsers;
7+
using DotJEM.Json.Index2.Documents.Fields;
8+
using DotJEM.Json.Index2.Management.Info;
9+
using DotJEM.Json.Index2.Management.Observables;
10+
using DotJEM.Json.Index2.Management.Snapshots;
11+
using DotJEM.Json.Index2.Management.Snapshots.Zip;
12+
using DotJEM.Json.Index2.Management.Source;
13+
using DotJEM.Json.Index2.Snapshots;
14+
using DotJEM.ObservableExtensions.InfoStreams;
15+
using DotJEM.Web.Scheduler;
16+
using Newtonsoft.Json.Linq;
17+
using NUnit.Framework;
18+
19+
namespace DotJEM.Json.Index2.Management.Test;
20+
21+
22+
23+
[TestFixture]
24+
public class JsonIndexManagerTest
25+
{
26+
[Test, Explicit]
27+
public async Task IndexWriterShouldNotBeDisposed()
28+
{
29+
using TestDirectory dir = new();
30+
31+
IJsonIndex index = new JsonIndexBuilder("Test")
32+
.UsingSimpleFileStorage(dir.Info.CreateSubdirectory("index").FullName)
33+
.WithFieldResolver(new FieldResolver("id", "type"))
34+
.WithSnapshoting()
35+
.Build();
36+
37+
IJsonDocumentSource source = new DummyDocumentSource();
38+
IWebTaskScheduler scheduler = new WebTaskScheduler();
39+
ISnapshotStrategy strategy = new ZipSnapshotStrategy(dir.Info.CreateSubdirectory("snapshot").FullName);
40+
IJsonIndexSnapshotManager snapshots = new JsonIndexSnapshotManager(index, strategy, scheduler, "60h");
41+
IJsonIndexManager manager = new JsonIndexManager(source, snapshots, index);
42+
43+
InfoStreamExceptionEvent? disposedEvent = null;
44+
manager.InfoStream
45+
.OfType<InfoStreamExceptionEvent>()
46+
.Where(@event => @event.Exception is ObjectDisposedException)
47+
.Subscribe(@event =>
48+
{
49+
Debug.WriteLine(@event.Exception);
50+
disposedEvent = @event;
51+
});
52+
53+
try
54+
{
55+
await manager.RunAsync();
56+
Debug.WriteLine("TEST STARTED");
57+
Stopwatch sw = Stopwatch.StartNew();
58+
while (sw.Elapsed < 10.Minutes() && disposedEvent == null)
59+
{
60+
Task result = Random.Shared.Next(100) switch
61+
{
62+
(>= 0 and <= 50) => DoAfterDelay(manager.TakeSnapshotAsync, 1.Seconds()),
63+
(> 50 and <= 100) => DoAfterDelay(manager.ResetIndexAsync, 1.Seconds()),
64+
_ => Task.CompletedTask
65+
};
66+
await result;
67+
}
68+
69+
async Task DoAfterDelay(Func<Task> action, TimeSpan? delay = null)
70+
{
71+
await Task.Delay(delay ?? Random.Shared.Next(1,5).Seconds());
72+
await action();
73+
}
74+
await manager.StopAsync();
75+
index.Close();
76+
}
77+
catch (Exception e)
78+
{
79+
Console.WriteLine(e);
80+
await manager.StopAsync();
81+
index.Close();
82+
throw;
83+
}
84+
85+
Assert.That(disposedEvent, Is.Null, () => disposedEvent?.Exception.ToString());
86+
}
87+
88+
89+
}
90+
91+
public class TestDirectory : IDisposable
92+
{
93+
public DirectoryInfo Info { get; }
94+
95+
public TestDirectory()
96+
{
97+
Info = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), $"TEST-{Guid.NewGuid():N}"));
98+
Debug.WriteLine("TEST DIR: " + Info.FullName);
99+
}
100+
101+
102+
public void Dispose()
103+
{
104+
try
105+
{
106+
Info.Delete(true);
107+
}
108+
catch (Exception e)
109+
{
110+
}
111+
}
112+
113+
~TestDirectory()
114+
{
115+
Dispose();
116+
}
117+
}
118+
public class DummyDocumentSource : IJsonDocumentSource
119+
{
120+
private readonly DocumentChangesStream observable = new();
121+
private readonly InfoStream<DummyDocumentSource> infoStream = new();
122+
123+
public IObservable<IJsonDocumentSourceEvent> DocumentChanges => observable;
124+
public IObservableValue<bool> Initialized { get; } = new ObservableValue<bool>();
125+
public IInfoStream InfoStream => infoStream;
126+
127+
private Faker faker = new Faker("en");
128+
private long gen;
129+
130+
private Task? runningTask;
131+
private CancellationTokenSource cancellationTokenSource = new ();
132+
private readonly string area = "Test";
133+
134+
public async Task StartAsync()
135+
{
136+
infoStream.WriteJsonSourceEvent(JsonSourceEventType.Starting, area, $"Ingest starting for storageArea '{area}'.");
137+
runningTask = Task.Run(async () =>
138+
{
139+
while (gen < 1_000_000 && !cancellationTokenSource.IsCancellationRequested)
140+
{
141+
if (!Initialized.Value)
142+
{
143+
infoStream.WriteJsonSourceEvent(JsonSourceEventType.Initializing, area, $"Initializing for storageArea '{area}'.");
144+
RunLoop();
145+
Initialized.Value = true;
146+
infoStream.WriteJsonSourceEvent(JsonSourceEventType.Initialized, area, $"Initializing for storageArea '{area}'.");
147+
}
148+
else
149+
{
150+
infoStream.WriteJsonSourceEvent(JsonSourceEventType.Updating, area, $"Checking updates for storageArea '{area}'.");
151+
RunLoop();
152+
infoStream.WriteJsonSourceEvent(JsonSourceEventType.Updated, area, $"Done checking updates for storageArea '{area}'.");
153+
}
154+
observable.Publish(new JsonDocumentSourceDigestCompleted(area));
155+
await Task.Delay(Random.Shared.Next(50, 250));
156+
}
157+
}, cancellationTokenSource.Token);
158+
159+
void RunLoop()
160+
{
161+
foreach (JObject json in Enumerable.Repeat(0, Random.Shared.Next(1, 100))
162+
.Select(_ => Guid.NewGuid())
163+
.Select(id => new
164+
{
165+
id,
166+
type = "Test",
167+
area,
168+
time = DateTime.Now,
169+
user = new
170+
{
171+
name = faker.Name.FirstName()
172+
}
173+
})
174+
.Select(JObject.FromObject))
175+
{
176+
observable.Publish(new JsonDocumentCreated(area, json, 10, new GenerationInfo(gen++, 1_000_000)));
177+
}
178+
}
179+
}
180+
181+
public async Task StopAsync()
182+
{
183+
cancellationTokenSource.Cancel();
184+
185+
if(runningTask != null)
186+
await runningTask.ConfigureAwait(false);
187+
188+
runningTask = null;
189+
cancellationTokenSource = new();
190+
infoStream.WriteJsonSourceEvent(JsonSourceEventType.Stopped, area, $"Stopping for storageArea '{area}'.");
191+
}
192+
193+
public void UpdateGeneration(string area, long generation)
194+
{
195+
gen = generation;
196+
}
197+
198+
public async Task ResetAsync()
199+
{
200+
gen = 0;
201+
}
202+
}

‎src/DotJEM.Json.Index2.Management/IJsonIndexManager.cs

+7-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ public interface IJsonIndexManager
2727
Task RunAsync();
2828
Task UpdateGenerationAsync(string area, long generation);
2929
Task ResetIndexAsync();
30+
Task StopAsync();
3031
}
3132

3233
public class JsonIndexManager : IJsonIndexManager
@@ -130,7 +131,12 @@ public async Task ResetIndexAsync()
130131
await jsonDocumentSource.ResetAsync().ConfigureAwait(false);
131132
await jsonDocumentSource.StartAsync().ConfigureAwait(false);
132133
}
133-
134+
135+
public async Task StopAsync()
136+
{
137+
await jsonDocumentSource.StopAsync().ConfigureAwait(false);
138+
}
139+
134140
private void CaptureChange(IJsonDocumentSourceEvent sourceEvent)
135141
{
136142
try

‎src/DotJEM.Json.Index2.Management/Tracking/IIngestProgressTracker.cs

+6-5
Original file line numberDiff line numberDiff line change
@@ -99,22 +99,23 @@ public void OnNext(IJsonDocumentSourceEvent value)
9999
{
100100
case JsonDocumentCreated created:
101101
observerTrackers.AddOrUpdate(value.Area,
102-
_ => throw new InvalidDataException(),
102+
_ => new StorageAreaIngestStateTracker(value.Area, JsonSourceEventType.Starting),
103103
(_, state) => state.UpdateState(created.Generation, created.Size));
104104
break;
105105
case JsonDocumentUpdated updated:
106-
observerTrackers.AddOrUpdate(value.Area,
107-
_ => throw new InvalidDataException(),
106+
observerTrackers.AddOrUpdate(value.Area,
107+
_ => new StorageAreaIngestStateTracker(value.Area, JsonSourceEventType.Starting),
108108
(_, state) => state.UpdateState(updated.Generation, updated.Size));
109109
break;
110110
case JsonDocumentDeleted deleted:
111111
observerTrackers.AddOrUpdate(value.Area,
112-
_ => throw new InvalidDataException(),
112+
_ => new StorageAreaIngestStateTracker(value.Area, JsonSourceEventType.Starting),
113113
(_, state) => state.UpdateState(deleted.Generation, deleted.Size));
114114
break;
115115
case JsonDocumentSourceDigestCompleted:
116116
observerTrackers.AddOrUpdate(value.Area,
117-
_ => throw new InvalidDataException(), (_, state) => state);
117+
_ => new StorageAreaIngestStateTracker(value.Area, JsonSourceEventType.Starting),
118+
(_, state) => state);
118119
break;
119120
case JsonDocumentSourceReset:
120121
break;

‎src/DotJEM.Json.Index2.Management/Writer/IJsonIndexWriter.cs

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public JsonIndexWriter(IJsonIndex index)
4040
this.index = index;
4141
this.mapper = index.Configuration.DocumentFactory;
4242
this.resolver = index.Configuration.FieldInformationManager;
43+
4344
throttledCommit = new ThrottledCommit(this);
4445
}
4546

‎src/DotJEM.Json.Index2.QueryParsers.Test/DotJEM.Json.Index2.QueryParsers.Test.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
</PropertyGroup>
1313

1414
<ItemGroup>
15+
<PackageReference Include="Bogus" Version="35.5.1" />
1516
<PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00016" />
1617
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
1718
<PackageReference Include="NUnit" Version="3.14.0" />

‎src/DotJEM.Json.Index2.QueryParsers.Test/SimplifiedLuceneQueryParserIntegrationTest.cs

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using DotJEM.Json.Index2.QueryParsers.Query;
55
using DotJEM.Json.Index2.Results;
66
using DotJEM.Json.Index2.Searching;
7+
using Lucene.Net.Analysis;
78
using Lucene.Net.Analysis.En;
89
using Lucene.Net.Analysis.Standard;
910
using Lucene.Net.Analysis.Util;

0 commit comments

Comments
 (0)