Skip to content

Commit c6bc149

Browse files
AndMuAndMu
authored
Merge pull request #8 from AndMu/feature/FixContainers
Feature/fix containers
2 parents bc1b0a9 + 9e8cfb3 commit c6bc149

File tree

74 files changed

+841
-722
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+841
-722
lines changed

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Adjustment/AdjustmentTests.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.IO;
44
using System.Threading.Tasks;
55
using Wikiled.Sentiment.TestLogic.Shared.Helpers;
6+
using Wikiled.Sentiment.Text.Extensions;
67
using Wikiled.Sentiment.Text.NLP;
78
using Wikiled.Sentiment.Text.Parser;
89
using Wikiled.Text.Analysis.Structure;
@@ -25,18 +26,20 @@ public async Task Adjusted()
2526
var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv");
2627
ISentimentDataHolder lexicon = SentimentDataHolder.Load(words);
2728
var text = "I Veto it";
28-
Document result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
29+
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
30+
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
2931
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
30-
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(result).Create();
32+
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(document).Create();
3133
Assert.AreEqual(1, review.CalculateRawRating().StarsRating);
3234
}
3335

3436
[Test]
3537
public async Task TestEmoticon()
3638
{
3739
var text = "EMOTICON_confused I do";
38-
Document result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
39-
var review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(result).Create();
40+
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
41+
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
42+
var review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(document).Create();
4043
Assert.AreEqual(1, review.CalculateRawRating().StarsRating);
4144
}
4245
}

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Containers/ContainersTests.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using NUnit.Framework;
3+
using System.Threading.Tasks;
4+
using Wikiled.Common.Utilities.Modules;
5+
using Wikiled.Sentiment.Analysis.Containers;
6+
using Wikiled.Sentiment.Text.Data.Review;
7+
using Wikiled.Sentiment.Text.NLP.Repair;
8+
using Wikiled.Sentiment.Text.Resources;
9+
using Wikiled.Text.Analysis.Structure;
10+
11+
namespace Wikiled.Sentiment.AcceptanceTests.Containers
12+
{
13+
[TestFixture]
14+
public class ContainersTests
15+
{
16+
[Test]
17+
public async Task Construct()
18+
{
19+
var configuration = new ConfigurationHandler();
20+
configuration.SetConfiguration("lexicons", "lexicons");
21+
configuration.StartingLocation = TestContext.CurrentContext.TestDirectory;
22+
23+
var builder = new ServiceCollection();
24+
builder.RegisterModule<LoggingModule>();
25+
builder.RegisterModule<CommonModule>();
26+
builder.RegisterModule(new SentimentMainModule());
27+
builder.RegisterModule(new SentimentServiceModule(configuration) { Lexicons = "." });
28+
var container = builder.BuildServiceProvider();
29+
30+
for (int i = 0; i < 2; i++)
31+
{
32+
using var scope = container.CreateScope();
33+
var session = scope.ServiceProvider.GetService<ISessionContainer>();
34+
var client = session.GetTesting();
35+
client.Init();
36+
37+
var result = await client
38+
.Process(new ParsingDocumentHolder(session.GetTextSplitter(),
39+
session.GetWordFactory(),
40+
session.Resolve<IContextSentenceRepairHandler>(),
41+
new Document("I like beer")))
42+
.ConfigureAwait(false);
43+
Assert.AreEqual(5, result.Adjustment.Rating.StarsRating);
44+
}
45+
}
46+
}
47+
}

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Helpers/TestHelper.cs‎

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,31 @@
1-
using System;
1+
using Microsoft.Extensions.DependencyInjection;
2+
using NUnit.Framework;
23
using System.Configuration;
34
using System.IO;
4-
using Microsoft.Extensions.DependencyInjection;
5-
using NUnit.Framework;
65
using Wikiled.Amazon.Logic;
6+
using Wikiled.Common.Utilities.Modules;
77
using Wikiled.Redis.Config;
8-
using Wikiled.Redis.Logic;
98
using Wikiled.Sentiment.Analysis.Containers;
109
using Wikiled.Sentiment.TestLogic.Shared.Helpers;
1110

1211
namespace Wikiled.Sentiment.AcceptanceTests.Helpers
1312
{
1413
public class TestHelper
1514
{
16-
private readonly Lazy<IRedisLink> redis;
17-
18-
private readonly Lazy<AmazonRepository> amazonRepository;
19-
2015
private readonly IGlobalContainer container;
2116

2217
public TestHelper(string server = "192.168.0.70", int port = 6373)
2318
{
24-
redis = new Lazy<IRedisLink>(() =>
25-
{
26-
var serverInstance = new RedisServer(new RedisConfiguration(server, port) { ServiceName = "Wikiled" });
27-
return serverInstance.Provider.GetService<IRedisLink>();
28-
});
19+
IServiceCollection service = new ServiceCollection();
20+
service.RegisterModule(new RedisServerModule(new RedisConfiguration(server, port) {ServiceName = "Wikiled"}));
21+
service.AddSingleton<AmazonRepository>();
2922

30-
amazonRepository = new Lazy<AmazonRepository>(() => new AmazonRepository(Redis));
3123
container = MainContainerFactory
32-
.Setup(new ServiceCollection())
33-
.SetupLocalCache()
34-
.Config(item => item.SetConfiguration("resources", Path.Combine(TestContext.CurrentContext.TestDirectory, ConfigurationManager.AppSettings["resources"])))
35-
.Splitter()
36-
.Create();
24+
.Setup(service)
25+
.SetupLocalCache()
26+
.Config(item => item.SetConfiguration("resources", Path.Combine(TestContext.CurrentContext.TestDirectory, ConfigurationManager.AppSettings["resources"])))
27+
.Splitter()
28+
.Create();
3729

3830
Reset();
3931
}
@@ -45,10 +37,9 @@ public void Reset()
4537
ContainerHelper = container.StartSession();
4638
}
4739

48-
public AmazonRepository AmazonRepository => amazonRepository.Value;
40+
public AmazonRepository AmazonRepository => ContainerHelper.Resolve<AmazonRepository>();
4941

5042
public ISessionContainer ContainerHelper { get; private set; }
5143

52-
public IRedisLink Redis => redis.Value;
5344
}
5445
}

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Helpers/TestRunner.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Wikiled.Sentiment.AcceptanceTests.Helpers.Data;
99
using Wikiled.Sentiment.Analysis.Containers;
1010
using Wikiled.Sentiment.Text.Data.Review;
11+
using Wikiled.Sentiment.Text.NLP.Repair;
1112
using Wikiled.Sentiment.Text.Parser;
1213

1314
namespace Wikiled.Sentiment.AcceptanceTests.Helpers
@@ -53,7 +54,10 @@ private async Task<IParsedDocumentHolder> ProcessReview(AmazonReview review)
5354
return null;
5455
}
5556

56-
return new ParsingDocumentHolder(Active.Resolve<ITextSplitter>(), doc);
57+
return new ParsingDocumentHolder(Active.Resolve<ITextSplitter>(),
58+
Active.GetWordFactory(),
59+
Active.Resolve<IContextSentenceRepairHandler>(),
60+
doc);
5761
}
5862
catch (Exception ex)
5963
{

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Sentiments/NlpTests.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Linq;
66
using System.Threading.Tasks;
77
using Wikiled.Sentiment.TestLogic.Shared.Helpers;
8+
using Wikiled.Sentiment.Text.Extensions;
89
using Wikiled.Sentiment.Text.NLP;
910
using Wikiled.Sentiment.Text.Parser;
1011
using Wikiled.Sentiment.Text.Structure;
@@ -48,9 +49,10 @@ public async Task TestReview(bool disableInvert)
4849
var lexicon = SentimentDataHolder.PopulateEmotionsData(data);
4950
ActualWordsHandler.InstanceOpen.Container.Context.DisableInvertors = disableInvert;
5051

51-
Document result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(txt)).ConfigureAwait(false);
52+
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(txt)).ConfigureAwait(false);
53+
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
5254
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
53-
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(result).Create();
55+
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(document).Create();
5456
MachineLearning.Mathematics.RatingData ratings = review.CalculateRawRating();
5557
Assert.AreEqual(1, review.Sentences.Count);
5658
Assert.AreEqual(disableInvert, ratings.IsPositive);
@@ -59,8 +61,9 @@ public async Task TestReview(bool disableInvert)
5961
[Test]
6062
public async Task TestPhrase()
6163
{
62-
Document result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest("In the forest I like perfect dinner")).ConfigureAwait(false);
63-
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(result).Create();
64+
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest("In the forest I like perfect dinner")).ConfigureAwait(false);
65+
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
66+
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve<Func<Document, IParsedReviewManager>>()(document).Create();
6467
Assert.AreEqual(4, review.ImportantWords.Count());
6568
}
6669
}

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Sentiments/RegressionTests.cs‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
using System.Reactive.Linq;
2-
using System.Threading.Tasks;
3-
using Microsoft.Extensions.Logging;
1+
using Microsoft.Extensions.Logging;
42
using NUnit.Framework;
3+
using System.IO;
4+
using System.Reactive.Linq;
5+
using System.Threading.Tasks;
56
using Wikiled.Amazon.Logic;
67
using Wikiled.Common.Logging;
78
using Wikiled.Sentiment.AcceptanceTests.Helpers;
89
using Wikiled.Sentiment.AcceptanceTests.Helpers.Data;
10+
using Wikiled.Sentiment.Analysis.Pipeline.Persistency;
911

1012
namespace Wikiled.Sentiment.AcceptanceTests.Sentiments
1113
{
@@ -17,19 +19,19 @@ public class RegressionTests
1719
private static readonly SentimentTestData[] testData =
1820
{
1921
new SentimentTestData("NotKnown", performance: "Total:<0> Positive:<0.000%> Negative:<0.000%> F1:<0.000> RMSE:NaN"),
20-
new SentimentTestData("B0002L5R78", 7581, 0, "Total:<7235> Positive:<77.420%> Negative:<71.951%> F1:<0.857> RMSE:1.52"),
22+
new SentimentTestData("B0002L5R78", 7581, 0, "Total:<7237> Positive:<77.504%> Negative:<71.545%> F1:<0.858> RMSE:1.52"),
2123

22-
new SentimentTestData("B00002EQCW", 228, 0, "Total:<212> Positive:<85.787%> Negative:<66.667%> F1:<0.911> RMSE:1.34"),
23-
new SentimentTestData("B000BAX50G", 288, 0, "Total:<266> Positive:<89.412%> Negative:<63.636%> F1:<0.936> RMSE:1.05"),
24-
new SentimentTestData("B000ERAON2", 440, 0, "Total:<414> Positive:<85.000%> Negative:<75.926%> F1:<0.901> RMSE:1.27"),
24+
new SentimentTestData("B00002EQCW", 228, 0, "Total:<212> Positive:<86.802%> Negative:<66.667%> F1:<0.917> RMSE:1.34"),
25+
new SentimentTestData("B000BAX50G", 288, 0, "Total:<266> Positive:<90.980%> Negative:<63.636%> F1:<0.945> RMSE:1.02"),
26+
new SentimentTestData("B000ERAON2", 440, 0, "Total:<414> Positive:<85.556%> Negative:<74.074%> F1:<0.903> RMSE:1.25"),
2527

26-
new SentimentTestData("B0026127Y8", 928, 0, "Total:<853> Positive:<70.078%> Negative:<76.543%> F1:<0.812> RMSE:1.48") { Category = ProductCategory.Video },
27-
new SentimentTestData("B009GN6F5Q", 472, 0, "Total:<390> Positive:<79.853%> Negative:<74.359%> F1:<0.837> RMSE:1.32") { Category = ProductCategory.Video },
28-
new SentimentTestData("B009CG8YJW", 418, 0, "Total:<345> Positive:<59.603%> Negative:<69.767%> F1:<0.727> RMSE:1.52") { Category = ProductCategory.Video },
28+
new SentimentTestData("B0026127Y8", 928, 0, "Total:<853> Positive:<70.596%> Negative:<74.074%> F1:<0.815> RMSE:1.47") { Category = ProductCategory.Video },
29+
new SentimentTestData("B009GN6F5Q", 472, 0, "Total:<390> Positive:<79.487%> Negative:<75.214%> F1:<0.836> RMSE:1.30") { Category = ProductCategory.Video },
30+
new SentimentTestData("B009CG8YJW", 418, 0, "Total:<345> Positive:<59.934%> Negative:<72.093%> F1:<0.731> RMSE:1.50") { Category = ProductCategory.Video },
2931

30-
new SentimentTestData("B00004SGFS", 381, 0, "Total:<375> Positive:<94.737%> Negative:<72.727%> F1:<0.960> RMSE:0.94") { Category = ProductCategory.Kitchen },
31-
new SentimentTestData("B000PYF768", 507, 0, "Total:<479> Positive:<90.433%> Negative:<75.000%> F1:<0.939> RMSE:1.03") { Category = ProductCategory.Kitchen },
32-
new SentimentTestData("B0000Z6JIW", 297, 0, "Total:<288> Positive:<78.439%> Negative:<94.737%> F1:<0.877> RMSE:1.33") { Category = ProductCategory.Kitchen }
32+
new SentimentTestData("B00004SGFS", 381, 0, "Total:<375> Positive:<94.737%> Negative:<75.758%> F1:<0.961> RMSE:0.94") { Category = ProductCategory.Kitchen },
33+
new SentimentTestData("B000PYF768", 507, 0, "Total:<479> Positive:<89.977%> Negative:<75.000%> F1:<0.936> RMSE:1.02") { Category = ProductCategory.Kitchen },
34+
new SentimentTestData("B0000Z6JIW", 297, 0, "Total:<288> Positive:<77.323%> Negative:<94.737%> F1:<0.870> RMSE:1.34") { Category = ProductCategory.Kitchen }
3335
};
3436

3537
[TestCaseSource(nameof(testData))]
@@ -44,7 +46,14 @@ public async Task RawSentimentDetection(SentimentTestData data)
4446
testing.DisableSvm = true;
4547
testing.TrackArff = true;
4648
testing.Init();
47-
await testing.Process(runner.Load()).LastOrDefaultAsync();
49+
using var persistency = runner.Active.Resolve<IPipelinePersistency>();
50+
persistency.Start(Path.Combine(TestContext.CurrentContext.TestDirectory, "Results", data.Product));
51+
persistency.Debug = true;
52+
await testing.Process(runner.Load()).ForEachAsync(
53+
item =>
54+
{
55+
//persistency.Save(item);
56+
});
4857
Assert.AreEqual(data.Errors, testing.Errors);
4958
Assert.AreEqual(data.Performance, testing.GetPerformanceDescription());
5059
}

‎src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Sentiments/SentimentTests.cs‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,13 @@
1414
using Wikiled.Sentiment.AcceptanceTests.Helpers.Data;
1515
using Wikiled.Sentiment.Analysis.Processing;
1616
using Wikiled.Sentiment.Analysis.Processing.Persistency;
17+
using Wikiled.Sentiment.TestLogic.Shared.Helpers;
1718
using Wikiled.Sentiment.Text.Data;
1819
using Wikiled.Sentiment.Text.Data.Review;
20+
using Wikiled.Sentiment.Text.Extensions;
1921
using Wikiled.Sentiment.Text.MachineLearning;
2022
using Wikiled.Sentiment.Text.NLP;
23+
using Wikiled.Sentiment.Text.NLP.Repair;
2124
using Wikiled.Sentiment.Text.Parser;
2225
using Wikiled.Text.Analysis.Structure;
2326

@@ -44,20 +47,26 @@ public void TearDown()
4447
public async Task SimpleTest()
4548
{
4649
var data = new XmlDataLoader(new NullLogger<XmlDataLoader>()).LoadOldXml(Path.Combine(TestContext.CurrentContext.TestDirectory, "data", "articles.xml"));
50+
4751
var negative = data.Load()
4852
.Where(item => item.Sentiment == SentimentClass.Negative)
4953
.Select(item => item.Data)
5054
.Repeat(15)
5155
.Select(
52-
item =>
53-
{
54-
item.Result.Stars = 1;
55-
item.Result.Text = item.Result.Text;
56-
item.Result.Id = Guid.NewGuid().ToString();
57-
return new ParsingDocumentHolder(TestHelper.Instance.ContainerHelper.GetTextSplitter(), item.Result);
58-
});
56+
item =>
57+
{
58+
item.Result.Stars = 1;
59+
item.Result.Text = item.Result.Text;
60+
item.Result.Id = Guid.NewGuid().ToString();
61+
62+
return new ParsingDocumentHolder(TestHelper.Instance.ContainerHelper.GetTextSplitter(),
63+
TestHelper.Instance.ContainerHelper.GetWordFactory(),
64+
TestHelper.Instance.ContainerHelper.Resolve<IContextSentenceRepairHandler>(),
65+
item.Result);
66+
});
5967

60-
var positive = data.Load().Where(item => item.Sentiment == SentimentClass.Positive)
68+
var positive = data.Load()
69+
.Where(item => item.Sentiment == SentimentClass.Positive)
6170
.Select(item => item.Data)
6271
.Repeat(15)
6372
.Select(
@@ -66,7 +75,11 @@ public async Task SimpleTest()
6675
item.Result.Stars = 5;
6776
item.Result.Text = item.Result.Text;
6877
item.Result.Id = Guid.NewGuid().ToString();
69-
return new ParsingDocumentHolder(TestHelper.Instance.ContainerHelper.GetTextSplitter(), item.Result);
78+
79+
return new ParsingDocumentHolder(TestHelper.Instance.ContainerHelper.GetTextSplitter(),
80+
TestHelper.Instance.ContainerHelper.GetWordFactory(),
81+
TestHelper.Instance.ContainerHelper.Resolve<IContextSentenceRepairHandler>(),
82+
item.Result);
7083
});
7184

7285
var trainingPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "training");
@@ -95,7 +108,8 @@ public async Task SimpleAmazonTest()
95108
var reviews = TestHelper.Instance.AmazonRepository.LoadProductReviews("B00005A0QX").ToEnumerable().ToArray();
96109
var review = reviews.First(item => item.User.Id == "AOJRUSTYHKT1T");
97110
var doc = await TestHelper.Instance.ContainerHelper.GetTextSplitter().Process(new ParseRequest(review.CreateDocument())).ConfigureAwait(false);
98-
var result = TestHelper.Instance.ContainerHelper.Resolve<Func<Document, IParsedReviewManager>>()(doc).Create();
111+
var document = doc.Construct(TestHelper.Instance.ContainerHelper.GetWordFactory());
112+
var result = TestHelper.Instance.ContainerHelper.Resolve<Func<Document, IParsedReviewManager>>()(document).Create();
99113
Assert.IsNotNull(result);
100114
Assert.AreEqual(1, result.Sentences.Count);
101115
var rating = result.CalculateRawRating();
@@ -110,7 +124,7 @@ public async Task SimpleAmazonTest()
110124
public async Task TestNull()
111125
{
112126
var doc = await TestHelper.Instance.ContainerHelper.GetTextSplitter().Process(new ParseRequest(new Document())).ConfigureAwait(false);
113-
Assert.AreEqual(0, doc.Sentences.Count);
127+
Assert.AreEqual(0, doc.Sentences.Length);
114128
}
115129

116130
[Test]

0 commit comments

Comments
 (0)