From 050b4e6a884f8983b455879c67c6601646a95bb4 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 05:01:22 +0530 Subject: [PATCH 01/16] ci-cd: reduce runtime & e2e suite logic to have manual trigger --- .../workflows/{test.yaml => e2e-suite.yaml} | 32 ++++++++++++------- .github/workflows/greetings.yaml | 20 +++++++++--- 2 files changed, 36 insertions(+), 16 deletions(-) rename .github/workflows/{test.yaml => e2e-suite.yaml} (75%) diff --git a/.github/workflows/test.yaml b/.github/workflows/e2e-suite.yaml similarity index 75% rename from .github/workflows/test.yaml rename to .github/workflows/e2e-suite.yaml index 3dddb521..b10e9d58 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -1,15 +1,25 @@ -name: Test Nixopus API +name: E2E Test Nixopus API on: push: branches: - - 'master' - - 'feat/develop' + - "master" + - "main" + - "feat/develop" paths: - - 'api/**' + - "api/**/*.go" + - "api/**/*.yaml" + - "api/**/*.yml" + - "api/**/*.sql" + workflow_dispatch: + inputs: + branch: + description: "Custom branch to run the E2E tests on" + required: true + default: "main" jobs: - test: + e2e-suite: runs-on: ubuntu-latest services: test-db: @@ -28,17 +38,17 @@ jobs: steps: - uses: actions/checkout@v4 - + - uses: actions/setup-go@v5 with: - go-version: '1.22' + go-version: "1.23.4" check-latest: true cache: true cache-dependency-path: api/go.sum - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Wait for PostgreSQL run: | for i in {1..10}; do @@ -51,7 +61,7 @@ jobs: done echo "PostgreSQL failed to start" exit 1 - + - name: Build and run tests env: DB_HOST: localhost @@ -60,4 +70,4 @@ jobs: DB_PASSWORD: nixopus DB_NAME: nixopus_test run: | - cd api && make test \ No newline at end of file + cd api && make test-all diff --git a/.github/workflows/greetings.yaml b/.github/workflows/greetings.yaml index d19553ce..4eab16dd 100644 --- a/.github/workflows/greetings.yaml +++ b/.github/workflows/greetings.yaml @@ -4,13 +4,23 @@ on: [pull_request_target, issues] jobs: greeting: + if: | + github.event_name == 'issues' || ( + github.event_name == 'pull_request_target' && + ( + github.event.pull_request.base.ref == 'main' || + github.event.pull_request.base.ref == 'master' || + contains(github.event.pull_request.base.ref, 'feat/') + + ) + ) runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: "Thank you for creating your first issue! We appreciate your contribution and will review it soon. Please ensure you've provided all necessary details and followed the issue template." - pr-message: "Thank you for your first pull request! Before we review, please ensure your code follows our quality standards: run tests, check formatting, and verify linting. We'll review your changes as soon as possible." \ No newline at end of file + - uses: actions/first-interaction@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + issue-message: "Thank you for creating your first issue! We appreciate your contribution and will review it soon. Please ensure you've provided all necessary details and followed the issue template." + pr-message: "Thank you for your first pull request! Before we review, please ensure your code follows our quality standards: run tests, check formatting, and verify linting. We'll review your changes as soon as possible." From c580920ea9f3e12c34ba212cfbb01d73a5f071b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?shravan=20=7C=7C=20=E0=A4=B6=E0=A5=8D=E0=A4=B0van?= Date: Fri, 11 Jul 2025 05:07:40 +0530 Subject: [PATCH 02/16] Update main.go --- api/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/main.go b/api/main.go index bce07c6c..934e57f6 100644 --- a/api/main.go +++ b/api/main.go @@ -33,3 +33,4 @@ func main() { log.Printf("Server starting on port %s", config.AppConfig.Port) log.Fatal(http.ListenAndServe(":"+config.AppConfig.Port, nil)) } + From 88803068a270301514111ee94110d3c84d8721f7 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 05:09:52 +0530 Subject: [PATCH 03/16] ci-cd: reduce runtime & e2e suite logic to have manual trigger --- .github/workflows/e2e-suite.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index b10e9d58..d74219b7 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -11,6 +11,16 @@ on: - "api/**/*.yaml" - "api/**/*.yml" - "api/**/*.sql" + pull_request: + branches: + - "master" + - "main" + - "feat/develop" + paths: + - "api/**/*.go" + - "api/**/*.yaml" + - "api/**/*.yml" + - "api/**/*.sql" workflow_dispatch: inputs: branch: From 2ed4ba91bfca5366b097e1ad252afb2f80a56570 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 05:36:17 +0530 Subject: [PATCH 04/16] ci-cd: server-run logic for e2e suite --- .github/workflows/e2e-suite.yaml | 74 +++++++++++++++++++++++++++++++- api/api/versions.json | 2 +- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index d74219b7..d6583166 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -72,7 +72,65 @@ jobs: echo "PostgreSQL failed to start" exit 1 - - name: Build and run tests + - name: Create .env file + run: | + cd api + cat > .env << EOF + DB_NAME=nixopus_test + USERNAME=nixopus + PASSWORD=nixopus + HOST_NAME=localhost + DB_PORT=5433 + SSL_MODE=disable + PORT=8080 + DOCKER_HOST=unix:///var/run/docker.sock + JWT_SECRET=test-jwt-secret-key-for-ci + REFRESH_TOKEN_SECRET=test-refresh-secret-key-for-ci + ENV=development + ALLOWED_ORIGIN=http://localhost:3000 + EOF + + - name: Install Air + run: | + go install github.com/air-verse/air@latest + + - name: Start API server with Air + env: + DB_HOST: localhost + DB_PORT: 5433 + DB_USER: nixopus + DB_PASSWORD: nixopus + DB_NAME: nixopus_test + PORT: 8080 + run: | + cd api + air & + echo $! > air.pid + + - name: Wait for API server + run: | + for i in {1..30}; do + if curl -f http://localhost:8080/api/v1/health; then + echo "API server is ready" + exit 0 + fi + echo "Waiting for API server... $i" + sleep 2 + done + echo "API server failed to start" + exit 1 + + - name: Run unit tests + env: + DB_HOST: localhost + DB_PORT: 5433 + DB_USER: nixopus + DB_PASSWORD: nixopus + DB_NAME: nixopus_test + run: | + cd api && go test -p 1 ./internal/features/... -v -count=1 + + - name: Run integration tests env: DB_HOST: localhost DB_PORT: 5433 @@ -80,4 +138,16 @@ jobs: DB_PASSWORD: nixopus DB_NAME: nixopus_test run: | - cd api && make test-all + cd api && go test -p 1 ./internal/tests/... -v -count=1 + + - name: Stop API server + if: always() + run: | + cd api + # Kill Air process if it exists + if [ -f air.pid ]; then + kill $(cat air.pid) || true + rm air.pid + fi + # Also kill any remaining processes on port 8080 + lsof -ti:8080 | xargs kill -9 || true diff --git a/api/api/versions.json b/api/api/versions.json index 22f4b10f..1377597c 100644 --- a/api/api/versions.json +++ b/api/api/versions.json @@ -3,7 +3,7 @@ { "version": "v1", "status": "active", - "release_date": "2025-07-05T21:25:27.672646+05:30", + "release_date": "2025-07-11T05:16:21.35541+05:30", "end_of_life": "0001-01-01T00:00:00Z", "changes": [ "Initial API version" From 20c25252b2e5f6d2865e3c0416e575fb85942e74 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 05:37:06 +0530 Subject: [PATCH 05/16] ci-cd: server-run logic for e2e suite --- .github/workflows/e2e-suite.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index d6583166..d80fa5ad 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -106,7 +106,7 @@ jobs: cd api air & echo $! > air.pid - + - name: Wait for API server run: | for i in {1..30}; do @@ -144,10 +144,8 @@ jobs: if: always() run: | cd api - # Kill Air process if it exists if [ -f air.pid ]; then kill $(cat air.pid) || true rm air.pid fi - # Also kill any remaining processes on port 8080 lsof -ti:8080 | xargs kill -9 || true From faa8ea5132d3161c329b1063af160561edd1e412 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 05:40:10 +0530 Subject: [PATCH 06/16] ci-cd: server-run logic for e2e suite --- .github/workflows/e2e-suite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index d80fa5ad..92c64f58 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -109,7 +109,7 @@ jobs: - name: Wait for API server run: | - for i in {1..30}; do + for i in {1..50}; do if curl -f http://localhost:8080/api/v1/health; then echo "API server is ready" exit 0 From 5b8f0b7567fda987e9a1c9142132f47ef54badeb Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 05:53:17 +0530 Subject: [PATCH 07/16] ci-cd: server-run logic for e2e suite --- .github/workflows/e2e-suite.yaml | 34 ++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 92c64f58..c9525888 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -90,11 +90,7 @@ jobs: ALLOWED_ORIGIN=http://localhost:3000 EOF - - name: Install Air - run: | - go install github.com/air-verse/air@latest - - - name: Start API server with Air + - name: Start API server with Go env: DB_HOST: localhost DB_PORT: 5433 @@ -104,20 +100,37 @@ jobs: PORT: 8080 run: | cd api - air & - echo $! > air.pid + echo "Contents of .env file:" + cat .env + echo "Starting server with go run..." + go run main.go > server.log 2>&1 & + echo $! > server.pid + echo "Server started with PID: $(cat server.pid)" + sleep 5 + echo "Server log output:" + cat server.log || echo "No server.log file found" - name: Wait for API server run: | + cd api for i in {1..50}; do + echo "Attempt $i: Testing connection to localhost:8080..." if curl -f http://localhost:8080/api/v1/health; then echo "API server is ready" exit 0 fi echo "Waiting for API server... $i" sleep 2 + + # Show server log every 10 attempts + if [ $((i % 10)) -eq 0 ]; then + echo "Server log at attempt $i:" + cat server.log || echo "No server.log file" + fi done echo "API server failed to start" + echo "Final server log:" + cat server.log || echo "No server.log file" exit 1 - name: Run unit tests @@ -144,8 +157,9 @@ jobs: if: always() run: | cd api - if [ -f air.pid ]; then - kill $(cat air.pid) || true - rm air.pid + # Kill server process if it exists + if [ -f server.pid ]; then + kill $(cat server.pid) || true + rm server.pid fi lsof -ti:8080 | xargs kill -9 || true From cad37d04a1fd6b607cfbad3166e08d4b442f44d6 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 06:03:23 +0530 Subject: [PATCH 08/16] ci-cd: server-run logic for e2e suite --- .github/workflows/e2e-suite.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index c9525888..794f20ff 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -46,6 +46,11 @@ jobs: --health-timeout 5s --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + steps: - uses: actions/checkout@v4 @@ -72,6 +77,19 @@ jobs: echo "PostgreSQL failed to start" exit 1 + - name: Wait for Redis + run: | + for i in {1..10}; do + if redis-cli -h localhost -p 6379 ping; then + echo "Redis is ready" + exit 0 + fi + echo "Waiting for Redis... $i" + sleep 2 + done + echo "Redis failed to start" + exit 1 + - name: Create .env file run: | cd api @@ -83,6 +101,7 @@ jobs: DB_PORT=5433 SSL_MODE=disable PORT=8080 + REDIS_URL=redis://localhost:6379 DOCKER_HOST=unix:///var/run/docker.sock JWT_SECRET=test-jwt-secret-key-for-ci REFRESH_TOKEN_SECRET=test-refresh-secret-key-for-ci From 0a2c002959b2d5db3e80a3dd17b276e8c9c95544 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 06:30:44 +0530 Subject: [PATCH 09/16] ci-cd: server-run logic for e2e suite --- .github/workflows/e2e-suite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 794f20ff..885e9b71 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -80,7 +80,7 @@ jobs: - name: Wait for Redis run: | for i in {1..10}; do - if redis-cli -h localhost -p 6379 ping; then + if nc -z localhost 6379; then echo "Redis is ready" exit 0 fi From dadfea367e700e569aa9c2ab307dcb7a55461303 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 19:11:14 +0530 Subject: [PATCH 10/16] fix: pipeline issue --- api/api/versions.json | 2 +- .../container/get_container_logs_test.go | 2 +- .../tests/container/get_container_test.go | 2 +- .../tests/container/list_containers_test.go | 2 +- api/internal/tests/domain/domain_flow_test.go | 4 +- api/internal/testutils/setup.go | 48 +++++++++++++++++++ 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/api/api/versions.json b/api/api/versions.json index 1377597c..bdc6c3f5 100644 --- a/api/api/versions.json +++ b/api/api/versions.json @@ -3,7 +3,7 @@ { "version": "v1", "status": "active", - "release_date": "2025-07-11T05:16:21.35541+05:30", + "release_date": "2025-07-11T18:51:33.216008+05:30", "end_of_life": "0001-01-01T00:00:00Z", "changes": [ "Initial API version" diff --git a/api/internal/tests/container/get_container_logs_test.go b/api/internal/tests/container/get_container_logs_test.go index 067889bc..7c54b446 100644 --- a/api/internal/tests/container/get_container_logs_test.go +++ b/api/internal/tests/container/get_container_logs_test.go @@ -11,7 +11,7 @@ import ( func TestGetContainerLogs(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/tests/container/get_container_test.go b/api/internal/tests/container/get_container_test.go index 0d11cc62..ffb8a400 100644 --- a/api/internal/tests/container/get_container_test.go +++ b/api/internal/tests/container/get_container_test.go @@ -11,7 +11,7 @@ import ( func TestGetContainer(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/tests/container/list_containers_test.go b/api/internal/tests/container/list_containers_test.go index 093dbe32..f99af4db 100644 --- a/api/internal/tests/container/list_containers_test.go +++ b/api/internal/tests/container/list_containers_test.go @@ -11,7 +11,7 @@ import ( func TestListContainers(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/tests/domain/domain_flow_test.go b/api/internal/tests/domain/domain_flow_test.go index c459ccb3..0a329829 100644 --- a/api/internal/tests/domain/domain_flow_test.go +++ b/api/internal/tests/domain/domain_flow_test.go @@ -11,7 +11,7 @@ import ( func TestCreateDomain(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -130,7 +130,7 @@ func TestCreateDomain(t *testing.T) { func TestGetDomains(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/testutils/setup.go b/api/internal/testutils/setup.go index d8a99b43..50499001 100644 --- a/api/internal/testutils/setup.go +++ b/api/internal/testutils/setup.go @@ -14,6 +14,8 @@ import ( authService "github.com/raghavyuva/nixopus-api/internal/features/auth/service" user_storage "github.com/raghavyuva/nixopus-api/internal/features/auth/storage" authTypes "github.com/raghavyuva/nixopus-api/internal/features/auth/types" + feature_flags_service "github.com/raghavyuva/nixopus-api/internal/features/feature-flags/service" + feature_flags_storage "github.com/raghavyuva/nixopus-api/internal/features/feature-flags/storage" "github.com/raghavyuva/nixopus-api/internal/features/logger" organization_service "github.com/raghavyuva/nixopus-api/internal/features/organization/service" organization_storage "github.com/raghavyuva/nixopus-api/internal/features/organization/storage" @@ -274,3 +276,49 @@ func (s *TestSetup) RegistrationHelper(email, password, username, orgName, orgDe return &authResponse, org, nil } + +// EnableAllFeaturesForOrg to enable all features for a test organization +func (s *TestSetup) EnableAllFeaturesForOrg(orgID uuid.UUID) error { + featureStorage := &feature_flags_storage.FeatureFlagStorage{DB: s.DB, Ctx: s.Ctx} + featureService := feature_flags_service.NewFeatureFlagService(featureStorage, s.Logger, s.Ctx) + + features := []string{ + "terminal", + "container", + "domain", + "file_manager", + "notifications", + "monitoring", + "github_connector", + "audit", + "self_hosted", + "deploy", + } + + for _, feature := range features { + req := types.UpdateFeatureFlagRequest{ + FeatureName: feature, + IsEnabled: true, + } + if err := featureService.UpdateFeatureFlag(orgID, req); err != nil { + return fmt.Errorf("failed to enable feature %s: %w", feature, err) + } + } + + return nil +} + +// GetTestAuthResponseWithAllFeatures to create a test user and organization with all features enabled +func (s *TestSetup) GetTestAuthResponseWithAllFeatures() (*authTypes.AuthResponse, *types.Organization, error) { + authResponse, org, err := s.GetTestAuthResponse() + if err != nil { + return nil, nil, err + } + + // Enable all features for the test organization + if err := s.EnableAllFeaturesForOrg(org.ID); err != nil { + return nil, nil, fmt.Errorf("failed to enable features for test org: %w", err) + } + + return authResponse, org, nil +} From 7ea01ce136f0fc2885f6c09dd140827e92538181 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 19:21:13 +0530 Subject: [PATCH 11/16] fix: pipeline issue --- .github/workflows/e2e-suite.yaml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 885e9b71..5f4b0c15 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -152,7 +152,7 @@ jobs: cat server.log || echo "No server.log file" exit 1 - - name: Run unit tests + - name: Run unit tests + intg tests env: DB_HOST: localhost DB_PORT: 5433 @@ -160,17 +160,7 @@ jobs: DB_PASSWORD: nixopus DB_NAME: nixopus_test run: | - cd api && go test -p 1 ./internal/features/... -v -count=1 - - - name: Run integration tests - env: - DB_HOST: localhost - DB_PORT: 5433 - DB_USER: nixopus - DB_PASSWORD: nixopus - DB_NAME: nixopus_test - run: | - cd api && go test -p 1 ./internal/tests/... -v -count=1 + cd api && make test-all - name: Stop API server if: always() From 777fa7587f405ccac2628ddd2f1db01db02d3ca1 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 20:21:54 +0530 Subject: [PATCH 12/16] fix: pipeline issue --- api/internal/tests/domain/domain_flow_test.go | 12 +++---- api/internal/testutils/feature_flag_test.go | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 api/internal/testutils/feature_flag_test.go diff --git a/api/internal/tests/domain/domain_flow_test.go b/api/internal/tests/domain/domain_flow_test.go index 0a329829..4aa9efe8 100644 --- a/api/internal/tests/domain/domain_flow_test.go +++ b/api/internal/tests/domain/domain_flow_test.go @@ -228,7 +228,7 @@ func TestGetDomains(t *testing.T) { func TestUpdateDomain(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -381,7 +381,7 @@ func TestUpdateDomain(t *testing.T) { func TestDeleteDomain(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -531,7 +531,7 @@ func TestDeleteDomain(t *testing.T) { func TestGenerateRandomSubDomain(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -623,7 +623,7 @@ func TestGenerateRandomSubDomain(t *testing.T) { func TestDomainsCRUDFlow(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -724,7 +724,7 @@ func TestDomainsCRUDFlow(t *testing.T) { func TestDomainPermissions(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -802,7 +802,7 @@ func TestDomainPermissions(t *testing.T) { func TestDomainErrorHandling(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/testutils/feature_flag_test.go b/api/internal/testutils/feature_flag_test.go new file mode 100644 index 00000000..dd8b8781 --- /dev/null +++ b/api/internal/testutils/feature_flag_test.go @@ -0,0 +1,35 @@ +package testutils + +import ( + "fmt" + "os" + "testing" +) + +func TestFeatureFlagSetup(t *testing.T) { + // Test the old way (without all features enabled) + t.Run("Standard setup", func(t *testing.T) { + setup := NewTestSetup() + user, org, err := setup.GetTestAuthResponse() + if err != nil { + t.Fatalf("failed to get test auth response: %v", err) + } + + fmt.Printf("Standard setup - User: %s, Org: %s\n", user.User.Email, org.Name) + }) + + // Test the new way (with all features enabled) + t.Run("All features enabled setup", func(t *testing.T) { + setup := NewTestSetup() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() + if err != nil { + t.Fatalf("failed to get test auth response with all features: %v", err) + } + + fmt.Printf("All features enabled setup - User: %s, Org: %s\n", user.User.Email, org.Name) + }) +} + +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} From c2258efc00a5612cafa1f54c5faad2c004ed05fb Mon Sep 17 00:00:00 2001 From: shravan20 Date: Fri, 11 Jul 2025 21:02:31 +0530 Subject: [PATCH 13/16] fix: pipeline issueclear --- api/api/versions.json | 2 +- .../tests/container/list_containers_test.go | 4 ++-- .../tests/feature-flags/feature_flags_test.go | 12 +++++------ api/internal/testutils/setup.go | 21 ++++++++++++++++--- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/api/api/versions.json b/api/api/versions.json index bdc6c3f5..18d0f547 100644 --- a/api/api/versions.json +++ b/api/api/versions.json @@ -3,7 +3,7 @@ { "version": "v1", "status": "active", - "release_date": "2025-07-11T18:51:33.216008+05:30", + "release_date": "2025-07-11T21:01:44.455809+05:30", "end_of_life": "0001-01-01T00:00:00Z", "changes": [ "Initial API version" diff --git a/api/internal/tests/container/list_containers_test.go b/api/internal/tests/container/list_containers_test.go index f99af4db..f90063c9 100644 --- a/api/internal/tests/container/list_containers_test.go +++ b/api/internal/tests/container/list_containers_test.go @@ -94,7 +94,7 @@ func TestListContainers(t *testing.T) { func TestListContainersWithSpecificContainer(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -117,7 +117,7 @@ func TestListContainersWithSpecificContainer(t *testing.T) { func TestListContainersErrorHandling(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/tests/feature-flags/feature_flags_test.go b/api/internal/tests/feature-flags/feature_flags_test.go index 77462e4e..0934117f 100644 --- a/api/internal/tests/feature-flags/feature_flags_test.go +++ b/api/internal/tests/feature-flags/feature_flags_test.go @@ -11,7 +11,7 @@ import ( func TestGetFeatureFlags(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -104,7 +104,7 @@ func TestGetFeatureFlags(t *testing.T) { func TestUpdateFeatureFlag(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -312,7 +312,7 @@ func TestUpdateFeatureFlag(t *testing.T) { func TestIsFeatureEnabled(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -469,7 +469,7 @@ func TestIsFeatureEnabled(t *testing.T) { func TestFeatureFlagsCRUDFlow(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -560,7 +560,7 @@ func TestFeatureFlagsCRUDFlow(t *testing.T) { func TestFeatureFlagPermissions(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } @@ -612,7 +612,7 @@ func TestFeatureFlagPermissions(t *testing.T) { func TestFeatureFlagErrorHandling(t *testing.T) { setup := testutils.NewTestSetup() - user, org, err := setup.GetTestAuthResponse() + user, org, err := setup.GetTestAuthResponseWithAllFeatures() if err != nil { t.Fatalf("failed to get test auth response: %v", err) } diff --git a/api/internal/testutils/setup.go b/api/internal/testutils/setup.go index 50499001..2195c148 100644 --- a/api/internal/testutils/setup.go +++ b/api/internal/testutils/setup.go @@ -121,6 +121,8 @@ func findMigrationsPath() string { "../../../migrations", "../../../../migrations", "migrations", + "../../migrations", + "../migrations", } for _, path := range paths { @@ -224,7 +226,9 @@ func (s *TestSetup) CreateTestUserAndOrg() (*types.User, *types.Organization, er } func (s *TestSetup) GetTestAuthResponse() (*authTypes.AuthResponse, *types.Organization, error) { - authResponse, org, err := s.RegistrationHelper("test@example.com", "Password123@", "testuser", "test-org", "Test organization", "admin") + // Make organization name unique to avoid conflicts + orgName := fmt.Sprintf("testuser's Team-%s", uuid.New().String()[:8]) + authResponse, org, err := s.RegistrationHelper("test@example.com", "Password123@", "testuser", orgName, "Test organization", "admin") if err != nil { return nil, nil, fmt.Errorf("failed to create test user: %w", err) } @@ -249,8 +253,8 @@ func (s *TestSetup) RegistrationHelper(email, password, username, orgName, orgDe // Create test organization org := &types.Organization{ ID: uuid.New(), - Name: "test-org", - Description: "Test organization", + Name: orgName, + Description: orgDescription, } if err := s.OrgStorage.CreateOrganization(*org); err != nil { @@ -274,6 +278,17 @@ func (s *TestSetup) RegistrationHelper(email, password, username, orgName, orgDe return nil, nil, fmt.Errorf("failed to add user to organization: %w", err) } + // Verify the user was actually added to the organization + belongs, err := s.UserStorage.UserBelongsToOrganization(authResponse.User.ID.String(), org.ID.String()) + if err != nil { + return nil, nil, fmt.Errorf("failed to verify user organization membership: %w", err) + } + if !belongs { + return nil, nil, fmt.Errorf("user was not successfully added to organization - membership verification failed") + } + + s.Logger.Log(logger.Info, "User organization membership verified", fmt.Sprintf("user_id=%s, org_id=%s", authResponse.User.ID.String(), org.ID.String())) + return &authResponse, org, nil } From c6c4731cb138df42c91b24b1a054c50360f4b2af Mon Sep 17 00:00:00 2001 From: shravan20 Date: Sat, 12 Jul 2025 07:35:16 +0530 Subject: [PATCH 14/16] fix: pipeline issue adding retries and wait --- .github/workflows/e2e-suite.yaml | 29 ++++++++++++++-- api/Makefile | 3 ++ api/api/versions.json | 2 +- api/internal/middleware/auth.go | 30 +++++++++++++++++ api/internal/tests/helper.go | 2 ++ api/internal/testutils/setup.go | 58 ++++++++++++++++++++++++++------ 6 files changed, 109 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 5f4b0c15..3651b043 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -134,17 +134,38 @@ jobs: cd api for i in {1..50}; do echo "Attempt $i: Testing connection to localhost:8080..." + + # Check if server process is still running + if [ -f server.pid ] && ! kill -0 $(cat server.pid) 2>/dev/null; then + echo "Server process died, checking logs:" + cat server.log || echo "No server.log file" + exit 1 + fi + + # Test health endpoint if curl -f http://localhost:8080/api/v1/health; then - echo "API server is ready" - exit 0 + echo "Health check passed, testing database connectivity..." + + # Test a simple auth endpoint to verify full readiness + response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/api/v1/auth/is-admin-registered) + if [ "$response" = "200" ]; then + echo "API server is fully ready (health + auth endpoints responding)" + + # Give additional time for any async initialization + sleep 3 + exit 0 + else + echo "Health passed but auth endpoint returned $response, server not fully ready" + fi fi + echo "Waiting for API server... $i" sleep 2 # Show server log every 10 attempts if [ $((i % 10)) -eq 0 ]; then echo "Server log at attempt $i:" - cat server.log || echo "No server.log file" + tail -20 server.log || echo "No server.log file" fi done echo "API server failed to start" @@ -159,6 +180,8 @@ jobs: DB_USER: nixopus DB_PASSWORD: nixopus DB_NAME: nixopus_test + TEST_TIMEOUT: 30s + TEST_RETRY_COUNT: 3 run: | cd api && make test-all diff --git a/api/Makefile b/api/Makefile index ddeb1fe3..07e26be4 100644 --- a/api/Makefile +++ b/api/Makefile @@ -11,6 +11,9 @@ test: test-all: @go test -p 1 ./... -v -count=1 +test-all-ci: + @go test -p 1 ./... -v -count=1 -timeout=15m -failfast + test-routes: @go test -p 1 ./internal/tests/routes/... -v -count=1 diff --git a/api/api/versions.json b/api/api/versions.json index 18d0f547..e0d3d1c7 100644 --- a/api/api/versions.json +++ b/api/api/versions.json @@ -3,7 +3,7 @@ { "version": "v1", "status": "active", - "release_date": "2025-07-11T21:01:44.455809+05:30", + "release_date": "2025-07-12T07:32:25.429981+05:30", "end_of_life": "0001-01-01T00:00:00Z", "changes": [ "Initial API version" diff --git a/api/internal/middleware/auth.go b/api/internal/middleware/auth.go index 8b0a23f6..a051cb5d 100644 --- a/api/internal/middleware/auth.go +++ b/api/internal/middleware/auth.go @@ -3,7 +3,9 @@ package middleware import ( "context" "fmt" + "log" "net/http" + "os" "strings" "time" @@ -25,6 +27,10 @@ func AuthMiddleware(next http.Handler, app *storage.App, cache *cache.Cache) htt token := r.Header.Get("Authorization") if token == "" { + //TODO: rmove log for debugging in CI + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: No authorization token provided for %s %s", r.Method, r.URL.Path) + } utils.SendErrorResponse(w, "No authorization token provided", http.StatusUnauthorized) return } @@ -41,12 +47,20 @@ func AuthMiddleware(next http.Handler, app *storage.App, cache *cache.Cache) htt user, err := verifyToken(token, app.Store.DB, ctx, cache) if err != nil { + // TODO: remove logs for debugging in CI + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: Token verification failed for %s %s: %v", r.Method, r.URL.Path, err) + } utils.SendErrorResponse(w, "Invalid authorization token", http.StatusUnauthorized) return } claims, err := getTokenClaims(token) if err != nil { + // TODO: remove after testing + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: Token claims extraction failed for %s %s: %v", r.Method, r.URL.Path, err) + } utils.SendErrorResponse(w, "Invalid token claims", http.StatusUnauthorized) return } @@ -55,6 +69,10 @@ func AuthMiddleware(next http.Handler, app *storage.App, cache *cache.Cache) htt twoFactorVerified, _ := claims["2fa_verified"].(bool) if twoFactorEnabled && !twoFactorVerified && !is2FAVerificationEndpoint(r.URL.Path) { + // TODO: Log for debugging in CI , remvoe after testing + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: 2FA required for %s %s", r.Method, r.URL.Path) + } utils.SendErrorResponse(w, "Two-factor authentication required", http.StatusForbidden) return } @@ -65,6 +83,10 @@ func AuthMiddleware(next http.Handler, app *storage.App, cache *cache.Cache) htt if !isAuthEndpoint(r.URL.Path) { organizationID := r.Header.Get("X-Organization-Id") if organizationID == "" { + // TODO: Log for debugging in CI , remove after testing + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: No organization ID provided for %s %s", r.Method, r.URL.Path) + } utils.SendErrorResponse(w, "No organization ID provided", http.StatusBadRequest) return } @@ -83,11 +105,19 @@ func AuthMiddleware(next http.Handler, app *storage.App, cache *cache.Cache) htt } belongsToOrg, err = userStorage.UserBelongsToOrganization(user.ID.String(), organizationID) if err != nil { + // TODO: remove log for debugging in CI + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: Error verifying org membership for %s %s: user=%s, org=%s, error=%v", r.Method, r.URL.Path, user.ID.String(), organizationID, err) + } utils.SendErrorResponse(w, "Error verifying organization membership", http.StatusInternalServerError) return } if !belongsToOrg { + // Log for debugging in CI + if os.Getenv("ENV") == "development" { + log.Printf("AUTH_DEBUG: User does not belong to organization for %s %s: user=%s, org=%s", r.Method, r.URL.Path, user.ID.String(), organizationID) + } utils.SendErrorResponse(w, "User does not belong to the specified organization", http.StatusForbidden) return } diff --git a/api/internal/tests/helper.go b/api/internal/tests/helper.go index 07b2ce2c..d4331ad4 100644 --- a/api/internal/tests/helper.go +++ b/api/internal/tests/helper.go @@ -2,6 +2,8 @@ package tests var baseURL = "http://localhost:8080/api/v1" + + func GetHealthURL() string { return baseURL + "/health" } diff --git a/api/internal/testutils/setup.go b/api/internal/testutils/setup.go index 2195c148..b85efbe1 100644 --- a/api/internal/testutils/setup.go +++ b/api/internal/testutils/setup.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -175,9 +176,19 @@ func NewTestSetup() *TestSetup { panic("ctx is nil - context not initialized") } - // Clean database before each test - if err := cleanDatabase(); err != nil { - panic(fmt.Sprintf("failed to clean database: %v", err)) + // Note: QUick validation to clean database before each test with retry logic for CI + maxRetries := 3 + var err error + for i := 0; i < maxRetries; i++ { + err = cleanDatabase() + if err == nil { + break + } + fmt.Printf("Database cleanup attempt %d failed: %v, retrying...\n", i+1, err) + time.Sleep(time.Duration(i+1) * time.Second) + } + if err != nil { + panic(fmt.Sprintf("failed to clean database after %d attempts: %v", maxRetries, err)) } l := logger.NewLogger() @@ -188,14 +199,25 @@ func NewTestSetup() *TestSetup { permStorage := &permissions_storage.PermissionStorage{DB: testDB, Ctx: ctx} roleStorage := &role_storage.RoleStorage{DB: testDB, Ctx: ctx} orgStorage := &organization_storage.OrganizationStore{DB: testDB, Ctx: ctx} - cache, err := cache.NewCache(getEnvOrDefault("REDIS_URL", "redis://localhost:6379")) + + // Initialize cache with retry logic + var cacheInstance *cache.Cache + for i := 0; i < maxRetries; i++ { + cacheInstance, err = cache.NewCache(getEnvOrDefault("REDIS_URL", "redis://localhost:6379")) + if err == nil { + break + } + fmt.Printf("Cache connection attempt %d failed: %v, retrying...\n", i+1, err) + time.Sleep(time.Duration(i+1) * time.Second) + } if err != nil { - panic(fmt.Sprintf("failed to create cache: %v", err)) + panic(fmt.Sprintf("failed to create cache after %d attempts: %v", maxRetries, err)) } + // Create services permService := permissions_service.NewPermissionService(store, ctx, l, permStorage) roleService := role_service.NewRoleService(store, ctx, l, roleStorage) - orgService := organization_service.NewOrganizationService(store, ctx, l, orgStorage, cache) + orgService := organization_service.NewOrganizationService(store, ctx, l, orgStorage, cacheInstance) authService := authService.NewAuthService(userStorage, l, permService, roleService, orgService, ctx) return &TestSetup{ @@ -310,16 +332,30 @@ func (s *TestSetup) EnableAllFeaturesForOrg(orgID uuid.UUID) error { "deploy", } + // Add retry logic for feature enabling in CI + maxRetries := 3 for _, feature := range features { - req := types.UpdateFeatureFlagRequest{ - FeatureName: feature, - IsEnabled: true, + var err error + for i := 0; i < maxRetries; i++ { + req := types.UpdateFeatureFlagRequest{ + FeatureName: feature, + IsEnabled: true, + } + err = featureService.UpdateFeatureFlag(orgID, req) + if err == nil { + break + } + s.Logger.Log(logger.Info, "Feature flag enable retry", fmt.Sprintf("feature=%s, attempt=%d, error=%v", feature, i+1, err)) + time.Sleep(time.Duration(i+1) * 100 * time.Millisecond) } - if err := featureService.UpdateFeatureFlag(orgID, req); err != nil { - return fmt.Errorf("failed to enable feature %s: %w", feature, err) + if err != nil { + return fmt.Errorf("failed to enable feature %s after %d attempts: %w", feature, maxRetries, err) } } + // Verify features were enabled with a small delay for consistency + time.Sleep(100 * time.Millisecond) + return nil } From fdcabc684844d577f251dcaafbd371850e436d92 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Sat, 12 Jul 2025 08:29:16 +0530 Subject: [PATCH 15/16] fix: pipeline issue adding retries and wait --- .github/workflows/e2e-suite.yaml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 3651b043..4ae5e1ee 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -31,9 +31,12 @@ on: jobs: e2e-suite: runs-on: ubuntu-latest + container: + image: golang:1.23.4-alpine + options: --user root services: test-db: - image: postgres:14-alpine + image: postgres:16-alpine env: POSTGRES_USER: nixopus POSTGRES_PASSWORD: nixopus @@ -54,15 +57,19 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: "1.23.4" - check-latest: true - cache: true - cache-dependency-path: api/go.sum + - name: Install dependencies + run: | + apk add --no-cache curl postgresql-client netcat-openbsd - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Set up Go cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('api/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - name: Wait for PostgreSQL run: | From cad94f2e4a7703b3cddde740dfde19cdccca4664 Mon Sep 17 00:00:00 2001 From: shravan20 Date: Sat, 12 Jul 2025 08:31:03 +0530 Subject: [PATCH 16/16] fix: pipeline issue adding retries and wait --- .github/workflows/e2e-suite.yaml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 4ae5e1ee..3651b043 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -31,12 +31,9 @@ on: jobs: e2e-suite: runs-on: ubuntu-latest - container: - image: golang:1.23.4-alpine - options: --user root services: test-db: - image: postgres:16-alpine + image: postgres:14-alpine env: POSTGRES_USER: nixopus POSTGRES_PASSWORD: nixopus @@ -57,19 +54,15 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install dependencies - run: | - apk add --no-cache curl postgresql-client netcat-openbsd - - - name: Set up Go cache - uses: actions/cache@v4 + - uses: actions/setup-go@v5 with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('api/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version: "1.23.4" + check-latest: true + cache: true + cache-dependency-path: api/go.sum + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Wait for PostgreSQL run: |