Merge pull request #248 from exceptionless/feature/repos

Updates to the new Foundatio.Repositories

Author: niemyjski

.gitignore

@@ -166,4 +166,5 @@ App_Data
 
 *.DS_Store
 *.orig
-Source/Api/Logs/log.txt
+Source/Api/Logs/*.txt
+*.nugetreferenceswitcher

Exceptionless.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
-VisualStudioVersion = 14.0.25123.0
+VisualStudioVersion = 14.0.25420.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Exceptionless.Core", "Source\Core\Exceptionless.Core.csproj", "{3E5B39D5-7ACD-486B-9F90-59116B67952D}"
 EndProject

Libraries/Create-Release.ps1

@@ -54,7 +54,6 @@ $apiConfig = [xml](Get-Content $webConfig)
 $apiConfig.SelectSingleNode('//appSettings/add[@key="BaseURL"]/@value').'#text' = 'http://localhost:50000/#'
 $apiConfig.SelectSingleNode('//appSettings/add[@key="EnableDailySummary"]/@value').'#text' = 'true'
 $apiConfig.SelectSingleNode('//system.web/compilation/@debug').'#text' = 'false'
-$apiConfig.SelectSingleNode('//system.web/customErrors/@mode').'#text' = 'RemoteOnly'
 
 # Copy settings from app web.config
 $appConfig = [xml](Get-Content "$releaseArtifactsDir\app\web.config")

Libraries/elasticsearch.prod.yml

@@ -0,0 +1,34 @@
+# multi node configuration
+cluster.name: exceptionless
+script.disable_dynamic: false
+index.number_of_shards: 3
+index.number_of_replicas: 1
+action.destructive_requires_name: true
+action.auto_create_index: +.marvel-*
+bootstrap.mlockall: true
+
+#path.data: /mnt/data/db
+#path.work: /path/to/work
+#path.logs: /mnt/data/log
+
+gateway.expected_nodes: 3
+discovery.zen.ping.multicast.enabled: false
+discovery.zen.minimum_master_nodes: 2
+discovery.zen.ping.unicast.hosts: [ '10.2.0.4', '10.2.0.5', '10.2.0.6' ]
+
+#transport.tcp.compress: true
+transport.tcp.port: 9300
+http.port: 9200
+http.jsonp.enable: true
+http.cors.enabled: true
+http.cors.allow-origin: http://dashboards.localhost.com
+
+# non-ssd configuration
+#index.merge.scheduler.max_thread_count: 1
+#index.translog.flush_threshold_size: 1gb
+
+#cloud:
+#    azure:
+#        storage:
+#            account: your_azure_storage_account
+#            key: your_azure_storage_key

Libraries/elasticsearch.yml

@@ -1,25 +1,15 @@
+# single node configuration
 cluster.name: exceptionless
-node.name: ex-es01
-#action.destructive_requires_name: true
-index.merge.scheduler.max_thread_count: 1
-index.translog.flush_threshold_size: 1gb
-#path.data: /mnt/data/db
-#path.work: /path/to/work
-#path.logs: /mnt/data/log
-bootstrap.mlockall: true
 script.disable_dynamic: false
-#gateway.expected_nodes: 3
+index.number_of_shards: 1
+index.number_of_replicas: 0
+action.auto_create_index: +.marvel-*
+node.local: true
+http.port: 9200
+transport.tcp.port: 9300
+bootstrap.mlockall: true
 discovery.zen.ping.multicast.enabled: false
-#discovery.zen.minimum_master_nodes: 2
-#discovery.zen.ping.unicast.hosts: [ '10.2.0.4', '10.2.0.5', '10.2.0.6' ]
-
-#transport.tcp.compress: true
-http.jsonp.enable: true
-http.cors.enabled: true
-http.cors.allow-origin: http://dashboards.localhost.com
 
-#cloud:
-#    azure:
-#        storage:
-#            account: your_azure_storage_account
-#            key: your_azure_storage_key
+# non-ssd configuration
+#index.merge.scheduler.max_thread_count: 1
+#index.translog.flush_threshold_size: 1gb

Source/Api/AppBuilder.cs

@@ -50,6 +50,7 @@ public static void Build(IAppBuilder app, Container container = null) {
             config.Formatters.JsonFormatter.SerializerSettings.Formatting = Formatting.Indented;
 
             SetupRouteConstraints(config);
+            container.RegisterSingleton(config);
             container.RegisterWebApiControllers(config);
 
             VerifyContainer(container);
@@ -76,8 +77,7 @@ public static void Build(IAppBuilder app, Container container = null) {
 
             EnableCors(config, app);
 
-            container.Bootstrap(config);
-            container.Bootstrap(app);
+            container.RunStartupActionsAsync().GetAwaiter().GetResult();
 
             app.UseWebApi(config);
             SetupSignalR(app, container, loggerFactory);
@@ -113,7 +113,7 @@ await workItemQueue.EnqueueAsync(new OrganizationNotificationWorkItem {
                 logger.Info("Jobs running out of process.");
                 return;
             }
-            
+
             new JobRunner(container.GetInstance<EventPostsJob>(), loggerFactory, initialDelay: TimeSpan.FromSeconds(2)).RunInBackground(token);
             new JobRunner(container.GetInstance<EventUserDescriptionsJob>(), loggerFactory, initialDelay: TimeSpan.FromSeconds(3)).RunInBackground(token);
             new JobRunner(container.GetInstance<EventNotificationsJob>(), loggerFactory, initialDelay: TimeSpan.FromSeconds(5)).RunInBackground(token);
@@ -205,7 +205,7 @@ private static async Task CreateSampleDataAsync(Container container) {
             await dataHelper.CreateDataAsync();
         }
 
-        public static Container CreateContainer(LoggerFactory loggerFactory, ILogger logger, bool includeInsulation = true) {
+        public static Container CreateContainer(ILoggerFactory loggerFactory, ILogger logger, bool includeInsulation = true) {
             var container = new Container();
             container.Options.AllowOverridingRegistrations = true;
             container.Options.DefaultScopedLifestyle = new WebApiRequestLifestyle();

Source/Api/Bootstrapper.cs

@@ -38,7 +38,7 @@ public static void RegisterServices(Container container, ILoggerFactory loggerFa
         }
 
         public class ApiMappings : Profile {
-            protected override void Configure() {
+            public ApiMappings() {
                 CreateMap<UserDescription, EventUserDescription>();
 
                 CreateMap<NewOrganization, Organization>();

Source/Api/Controllers/AdminController.cs

@@ -10,6 +10,7 @@
 using Exceptionless.Core.Models.WorkItems;
 using Exceptionless.Core.Queues.Models;
 using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Repositories.Configuration;
 using Foundatio.Jobs;
 using Foundatio.Messaging;
 using Foundatio.Queues;
@@ -20,13 +21,15 @@ namespace Exceptionless.Api.Controllers {
     [Authorize(Roles = AuthorizationRoles.GlobalAdmin)]
     [ApiExplorerSettings(IgnoreApi = true)]
     public class AdminController : ExceptionlessApiController {
+        private readonly ExceptionlessElasticConfiguration _configuration;
         private readonly IFileStorage _fileStorage;
         private readonly IMessagePublisher _messagePublisher;
         private readonly IOrganizationRepository _organizationRepository;
         private readonly IQueue<EventPost> _eventPostQueue;
         private readonly IQueue<WorkItemData> _workItemQueue;
 
-        public AdminController(IFileStorage fileStorage, IMessagePublisher messagePublisher, IOrganizationRepository organizationRepository, IQueue<EventPost> eventPostQueue, IQueue<WorkItemData> workItemQueue) {
+        public AdminController(ExceptionlessElasticConfiguration configuration, IFileStorage fileStorage, IMessagePublisher messagePublisher, IOrganizationRepository organizationRepository, IQueue<EventPost> eventPostQueue, IQueue<WorkItemData> workItemQueue) {
+            _configuration = configuration;
             _fileStorage = fileStorage;
             _messagePublisher = messagePublisher;
             _organizationRepository = organizationRepository;
@@ -93,6 +96,9 @@ public async Task<IHttpActionResult> RequeueAsync(string path = null, bool archi
         [Route("maintenance/{name:minlength(1)}")]
         public async Task<IHttpActionResult> RunJobAsync(string name) {
             switch (name.ToLower()) {
+                case "indexes":
+                    await _configuration.ConfigureIndexesAsync(beginReindexingOutdated: false);
+                    break;
                 case "update-organization-plans":
                     await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { UpgradePlans = true });
                     break;

Source/Api/Controllers/AuthController.cs

@@ -18,6 +18,7 @@
 using FluentValidation;
 using Foundatio.Caching;
 using Foundatio.Logging;
+using Foundatio.Utility;
 using Newtonsoft.Json.Linq;
 using OAuth2.Client;
 using OAuth2.Client.Impl;
@@ -32,7 +33,7 @@ public class AuthController : ExceptionlessApiController {
 	    private readonly IOrganizationRepository _organizationRepository;
         private readonly IUserRepository _userRepository;
         private readonly ITokenRepository _tokenRepository;
-        private readonly ICacheClient _cacheClient;
+        private readonly ICacheClient _cache;
         private readonly IMailer _mailer;
         private readonly ILogger _logger;
 
@@ -43,7 +44,7 @@ public AuthController(IOrganizationRepository organizationRepository, IUserRepos
 	        _organizationRepository = organizationRepository;
             _userRepository = userRepository;
             _tokenRepository = tokenRepository;
-            _cacheClient = new ScopedCacheClient(cacheClient, "auth");
+            _cache = new ScopedCacheClient(cacheClient, "Auth");
             _mailer = mailer;
             _logger = logger;
         }
@@ -81,11 +82,11 @@ public async Task<IHttpActionResult> LoginAsync(LoginModel model) {
 
             // Only allow 5 password attempts per 15 minute period.
             string userLoginAttemptsCacheKey = $"user:{model.Email}:attempts";
-            long userLoginAttempts = await _cacheClient.IncrementAsync(userLoginAttemptsCacheKey, 1, DateTime.UtcNow.Ceiling(TimeSpan.FromMinutes(15)));
+            long userLoginAttempts = await _cache.IncrementAsync(userLoginAttemptsCacheKey, 1, SystemClock.UtcNow.Ceiling(TimeSpan.FromMinutes(15)));
 
             // Only allow 15 login attempts per 15 minute period by a single ip.
             string ipLoginAttemptsCacheKey = $"ip:{Request.GetClientIpAddress()}:attempts";
-            long ipLoginAttempts = await _cacheClient.IncrementAsync(ipLoginAttemptsCacheKey, 1, DateTime.UtcNow.Ceiling(TimeSpan.FromMinutes(15)));
+            long ipLoginAttempts = await _cache.IncrementAsync(ipLoginAttemptsCacheKey, 1, SystemClock.UtcNow.Ceiling(TimeSpan.FromMinutes(15)));
 
             if (userLoginAttempts > 5) {
                 _logger.Error().Message("Login denied for \"{0}\" for the {1} time.", model.Email, userLoginAttempts).Tag("Login").Identity(model.Email).SetActionContext(ActionContext).Write();
@@ -136,7 +137,7 @@ public async Task<IHttpActionResult> LoginAsync(LoginModel model) {
             if (!String.IsNullOrEmpty(model.InviteToken))
                 await AddInvitedUserToOrganizationAsync(model.InviteToken, user);
 
-            await _cacheClient.RemoveAsync(userLoginAttemptsCacheKey);
+            await _cache.RemoveAsync(userLoginAttemptsCacheKey);
 
             _logger.Info().Message("\"{0}\" logged in.", user.EmailAddress).Tag("Login").Identity(user.EmailAddress).Property("User", user).SetActionContext(ActionContext).Write();
             return Ok(new TokenResult { Token = await GetTokenAsync(user) });
@@ -186,7 +187,7 @@ public async Task<IHttpActionResult> SignupAsync(SignupModel model) {
             bool hasValidInviteToken = !String.IsNullOrWhiteSpace(model.InviteToken) && await _organizationRepository.GetByInviteTokenAsync(model.InviteToken) != null;
             if (!hasValidInviteToken) {
                 // Only allow 10 signups per hour period by a single ip.
-                long ipSignupAttempts = await _cacheClient.IncrementAsync(ipSignupAttemptsCacheKey, 1, DateTime.UtcNow.Ceiling(TimeSpan.FromHours(1)));
+                long ipSignupAttempts = await _cache.IncrementAsync(ipSignupAttemptsCacheKey, 1, SystemClock.UtcNow.Ceiling(TimeSpan.FromHours(1)));
                 if (ipSignupAttempts > 10) {
                     _logger.Error().Message("Signup denied for \"{0}\" for the {1} time.", model.Email, ipSignupAttempts).Tag("Signup").Identity(model.Email).SetActionContext(ActionContext).Write();
                     return BadRequest();
@@ -335,7 +336,7 @@ public async Task<IHttpActionResult> IsEmailAddressAvailableAsync(string email)
 
             // Only allow 3 checks attempts per hour period by a single ip.
             string ipEmailAddressAttemptsCacheKey = $"ip:{Request.GetClientIpAddress()}:email:attempts";
-            long attempts = await _cacheClient.IncrementAsync(ipEmailAddressAttemptsCacheKey, 1, DateTime.UtcNow.Ceiling(TimeSpan.FromHours(1)));
+            long attempts = await _cache.IncrementAsync(ipEmailAddressAttemptsCacheKey, 1, SystemClock.UtcNow.Ceiling(TimeSpan.FromHours(1)));
 
             if (attempts > 3 || await _userRepository.GetByEmailAddressAsync(email) == null)
                 return StatusCode(HttpStatusCode.NoContent);
@@ -594,15 +595,15 @@ private Task ChangePasswordAsync(User user, string password) {
 
         private async Task<string> GetTokenAsync(User user) {
             var userTokens = await _tokenRepository.GetByUserIdAsync(user.Id);
-            var validAccessToken = userTokens.Documents.FirstOrDefault(t => (!t.ExpiresUtc.HasValue || t.ExpiresUtc > DateTime.UtcNow) && t.Type == TokenType.Access);
+            var validAccessToken = userTokens.Documents.FirstOrDefault(t => (!t.ExpiresUtc.HasValue || t.ExpiresUtc > SystemClock.UtcNow) && t.Type == TokenType.Access);
             if (validAccessToken != null)
                 return validAccessToken.Id;
 
             var token = await _tokenRepository.AddAsync(new Token {
                 Id = Core.Extensions.StringExtensions.GetNewToken(),
                 UserId = user.Id,
-                CreatedUtc = DateTime.UtcNow,
-                ModifiedUtc = DateTime.UtcNow,
+                CreatedUtc = SystemClock.UtcNow,
+                ModifiedUtc = SystemClock.UtcNow,
                 CreatedBy = user.Id,
                 Type = TokenType.Access
             });

Source/Api/Controllers/Base/ExceptionlessApiController.cs

@@ -2,19 +2,19 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
-using System.Text;
 using System.Threading.Tasks;
 using System.Web.Http;
 using System.Web.Http.Results;
 using Exceptionless.Api.Extensions;
 using Exceptionless.Api.Security;
 using Exceptionless.Api.Utility;
-using Exceptionless.Core.Extensions;
 using Exceptionless.Api.Utility.Results;
 using Exceptionless.Core.Repositories;
 using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories.Queries;
 using Exceptionless.DateTimeExtensions;
 using Foundatio.Repositories.Models;
+using Foundatio.Utility;
 
 namespace Exceptionless.Api.Controllers {
     [RequireHttpsExceptLocal]
@@ -49,7 +49,7 @@ protected virtual TimeInfo GetTimeInfo(string time, string offset, DateTime? min
             var utcOffset = GetOffset(offset);
 
             // range parsing needs to be based on the user's local time.
-            var localRange = DateTimeRange.Parse(time, DateTime.UtcNow.Add(utcOffset));
+            var localRange = DateTimeRange.Parse(time, SystemClock.UtcNow.Add(utcOffset));
             var utcRange = localRange != DateTimeRange.Empty ? localRange.Subtract(utcOffset) : localRange;
 
             if (utcRange.UtcStart < minimumUtcStartDate.GetValueOrDefault())
@@ -130,29 +130,35 @@ public ICollection<string> GetAssociatedOrganizationIds() {
             return Request.GetAssociatedOrganizationIds();
         }
 
-        public async Task<ICollection<Organization>> GetAssociatedOrganizationsAsync(IOrganizationRepository repository) {
+        private static readonly IReadOnlyCollection<Organization> EmptyOrganizations = new List<Organization>(0).AsReadOnly();
+        public async Task<IReadOnlyCollection<Organization>> GetAssociatedActiveOrganizationsAsync(IOrganizationRepository repository) {
             if (repository == null)
-                return null;
+                throw new ArgumentNullException(nameof(repository));
 
-            return (await repository.GetByIdsAsync(GetAssociatedOrganizationIds(), true)).Documents;
-        }
-        
-        public string BuildSystemFilter(ICollection<Organization> organizations, string filter, bool usesPremiumFeatures, string retentionDateFieldName = "date") {
-            if (HasOrganizationOrProjectOrStackFilter(filter) && Request.IsGlobalAdmin())
-                return null;
-            
-            var allowedOrganizations = organizations.Where(o => !o.IsSuspended && (o.HasPremiumFeatures || (!o.HasPremiumFeatures && !usesPremiumFeatures))).ToList();
-            if (allowedOrganizations.Count == 0)
-                return "organization:none";
+            var ids = GetAssociatedOrganizationIds();
+            if (ids.Count == 0)
+                return EmptyOrganizations;
 
-            return allowedOrganizations.BuildRetentionFilter(retentionDateFieldName);
+            var organizations = await repository.GetByIdsAsync(ids, true);
+            return organizations.Where(o => !o.IsSuspended).ToList().AsReadOnly();
         }
-        
-        private bool HasOrganizationOrProjectOrStackFilter(string filter) {
+
+        protected bool ShouldApplySystemFilter(IExceptionlessSystemFilterQuery sf, string filter) {
+            // Apply filter to non admin user.
+            if (!Request.IsGlobalAdmin())
+                return true;
+
+            // Apply filter as it's scoped via a controller action.
+            if (!sf.IsUserOrganizationsFilter)
+                return true;
+
+            // Empty user filter
             if (String.IsNullOrEmpty(filter))
-                return false;
+                return true;
 
-            return filter.Contains("organization:") || filter.Contains("project:") || filter.Contains("stack:");
+            // Used for impersonating a user. Only skip the filter if it contains an org, project or stack.
+            bool hasOrganizationOrProjectOrStackFilter = filter.Contains("organization:") || filter.Contains("project:") || filter.Contains("stack:");
+            return !hasOrganizationOrProjectOrStackFilter;
         }
 
         protected StatusCodeActionResult StatusCodeWithMessage(HttpStatusCode statusCode, string message, string reason = null) {
@@ -195,11 +201,11 @@ public OkWithHeadersContentResult<T> OkWithHeaders<T>(T content, IEnumerable<Key
             return new OkWithHeadersContentResult<T>(content, this, headers);
         }
 
-        public OkWithResourceLinks<TEntity> OkWithResourceLinks<TEntity>(ICollection<TEntity> content, bool hasMore, Func<TEntity, string> pagePropertyAccessor = null, IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers = null, bool isDescending = false) where TEntity : class {
+        public OkWithResourceLinks<TEntity> OkWithResourceLinks<TEntity>(IEnumerable<TEntity> content, bool hasMore, Func<TEntity, string> pagePropertyAccessor = null, IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers = null, bool isDescending = false) where TEntity : class {
             return new OkWithResourceLinks<TEntity>(content, this, hasMore, null, pagePropertyAccessor, headers, isDescending);
         }
 
-        public OkWithResourceLinks<TEntity> OkWithResourceLinks<TEntity>(ICollection<TEntity> content, bool hasMore, int page, long? total = null, IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers = null) where TEntity : class {
+        public OkWithResourceLinks<TEntity> OkWithResourceLinks<TEntity>(IEnumerable<TEntity> content, bool hasMore, int page, long? total = null, IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers = null) where TEntity : class {
             return new OkWithResourceLinks<TEntity>(content, this, hasMore, page, total);
         }