From 40e710cd586e1af1a9aaf890740d87a91aa21845 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michaela=20Pa=C5=99ilov=C3=A1?= <mparilov@redhat.com>
Date: Mon, 30 Mar 2026 19:26:43 +0200
Subject: [PATCH] HYPERFLEET-659 - fix: return HTTP 503 when database is
 unreachable

---
 pkg/db/transaction_middleware.go      |  28 +++-
 pkg/db/transaction_middleware_test.go | 184 ++++++++++++++++++++++++++
 pkg/errors/errors.go                  |   4 +
 pkg/errors/errors_test.go             |   8 ++
 pkg/services/generic.go               |   7 +-
 pkg/services/util.go                  |  13 ++
 6 files changed, 240 insertions(+), 4 deletions(-)

diff --git a/pkg/db/transaction_middleware.go b/pkg/db/transaction_middleware.go
index b0fb93ce..fa04cda0 100755
--- a/pkg/db/transaction_middleware.go
+++ b/pkg/db/transaction_middleware.go
@@ -2,6 +2,9 @@ package db
 
 import (
 	"context"
+	stderrors "errors"
+	"io"
+	"net"
 	"net/http"
 	"time"
 
@@ -47,8 +50,13 @@ func TransactionMiddleware(next http.Handler, connection SessionFactory, request
 			ctx, err = NewContext(ctx, connection)
 			if err != nil {
 				logger.WithError(ctx, err).Error("Could not create transaction")
-				// use default error to avoid exposing internals to users
-				serviceErr := errors.GeneralError("")
+				var serviceErr *errors.ServiceError
+				if IsDBConnectionError(err) {
+					serviceErr = errors.ServiceUnavailable("Database connection unavailable")
+				} else {
+					// use default error to avoid exposing internals to users
+					serviceErr = errors.GeneralError("")
+				}
 				traceID, _ := logger.GetRequestID(ctx)
 				response.WriteProblemDetailsResponse(w, r, serviceErr.HTTPCode, serviceErr.AsProblemDetails(r.URL.Path, traceID))
 				return
@@ -65,3 +73,19 @@ func TransactionMiddleware(next http.Handler, connection SessionFactory, request
 		}
 	})
 }
+
+// IsDBConnectionError indicates whether err is an infrastructure failure
+// (network unreachable, connection refused, connection dropped)
+func IsDBConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var netErr *net.OpError
+	if stderrors.As(err, &netErr) {
+		return true
+	}
+	if stderrors.Is(err, io.EOF) || stderrors.Is(err, io.ErrUnexpectedEOF) {
+		return true
+	}
+	return false
+}
diff --git a/pkg/db/transaction_middleware_test.go b/pkg/db/transaction_middleware_test.go
index 2ca18c3b..2060de92 100644
--- a/pkg/db/transaction_middleware_test.go
+++ b/pkg/db/transaction_middleware_test.go
@@ -1,10 +1,65 @@
 package db
 
 import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"net/http/httptest"
+	"syscall"
 	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	. "github.com/onsi/gomega"
+	postgres "gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	gormlogger "gorm.io/gorm/logger"
+
+	"github.com/openshift-hyperfleet/hyperfleet-api/pkg/config"
 )
 
+type dbSessionFactory struct {
+	gormDB *gorm.DB
+	sqlDB  *sql.DB
+}
+
+func (f *dbSessionFactory) Init(*config.DatabaseConfig)                             {}
+func (f *dbSessionFactory) New(_ context.Context) *gorm.DB                          { return f.gormDB }
+func (f *dbSessionFactory) CheckConnection() error                                  { return nil }
+func (f *dbSessionFactory) Close() error                                            { return nil }
+func (f *dbSessionFactory) ResetDB()                                                {}
+func (f *dbSessionFactory) NewListener(_ context.Context, _ string, _ func(string)) {}
+func (f *dbSessionFactory) GetAdvisoryLockTimeout() int                             { return 300 }
+func (f *dbSessionFactory) DirectDB() *sql.DB                                       { return f.sqlDB }
+
+func newDBSessionFactory(t *testing.T, setupMock func(sqlmock.Sqlmock)) *dbSessionFactory {
+	t.Helper()
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("failed to create sqlmock: %v", err)
+	}
+	setupMock(mock)
+
+	gormDB, err := gorm.Open(postgres.New(postgres.Config{Conn: db}), &gorm.Config{
+		Logger: gormlogger.Discard,
+	})
+	if err != nil {
+		t.Fatalf("failed to open GORM: %v", err)
+	}
+
+	t.Cleanup(func() {
+		db.Close()
+		if err := mock.ExpectationsWereMet(); err != nil {
+			t.Fatalf("unmet sqlmock expectations: %v", err)
+		}
+	})
+	return &dbSessionFactory{gormDB: gormDB, sqlDB: db}
+}
+
 func TestIsWriteMethod_StandardHTTPMethods(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -63,3 +118,132 @@ func TestIsWriteMethod_NonStandardMethods(t *testing.T) {
 		})
 	}
 }
+
+func TestTransactionMiddleware_DBUnavailable(t *testing.T) {
+	tests := []struct {
+		setupMock      func(sqlmock.Sqlmock)
+		name           string
+		expectedType   string
+		expectedCode   string
+		expectedDetail string
+		expectedStatus int
+	}{
+		{
+			name: "DB unreachable (connection refused) - returns 503",
+			setupMock: func(mock sqlmock.Sqlmock) {
+				mock.ExpectBegin().WillReturnError(
+					&net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED},
+				)
+			},
+			expectedStatus: http.StatusServiceUnavailable,
+			expectedType:   "https://api.hyperfleet.io/errors/service-unavailable",
+			expectedCode:   "HYPERFLEET-SVC-001",
+			expectedDetail: "Database connection unavailable",
+		},
+		{
+			name: "DB unreachable (connection dropped) - returns 503",
+			setupMock: func(mock sqlmock.Sqlmock) {
+				mock.ExpectBegin().WillReturnError(io.EOF)
+			},
+			expectedStatus: http.StatusServiceUnavailable,
+			expectedType:   "https://api.hyperfleet.io/errors/service-unavailable",
+			expectedCode:   "HYPERFLEET-SVC-001",
+			expectedDetail: "Database connection unavailable",
+		},
+		{
+			name: "internal DB error (permission denied) - returns 500",
+			setupMock: func(mock sqlmock.Sqlmock) {
+				mock.ExpectBegin().WillReturnError(errors.New("permission denied for table clusters"))
+			},
+			expectedStatus: http.StatusInternalServerError,
+			expectedType:   "https://api.hyperfleet.io/errors/internal-error",
+			expectedCode:   "HYPERFLEET-INT-001",
+			expectedDetail: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			RegisterTestingT(t)
+
+			factory := newDBSessionFactory(t, tt.setupMock)
+			next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			})
+
+			handler := TransactionMiddleware(next, factory, 0)
+			req := httptest.NewRequest(http.MethodPost, "/api/hyperfleet/v1/clusters", nil)
+			rec := httptest.NewRecorder()
+			handler.ServeHTTP(rec, req)
+
+			Expect(rec.Code).To(Equal(tt.expectedStatus))
+			Expect(rec.Header().Get("Content-Type")).To(Equal("application/problem+json"))
+
+			var body struct {
+				Code   *string `json:"code"`
+				Detail string  `json:"detail"`
+				Type   string  `json:"type"`
+				Status int     `json:"status"`
+			}
+			Expect(json.NewDecoder(rec.Body).Decode(&body)).To(Succeed())
+			Expect(body.Type).To(Equal(tt.expectedType))
+			Expect(body.Status).To(Equal(tt.expectedStatus))
+			Expect(body.Code).NotTo(BeNil())
+			Expect(*body.Code).To(Equal(tt.expectedCode))
+			if tt.expectedDetail != "" {
+				Expect(body.Detail).To(Equal(tt.expectedDetail))
+			}
+		})
+	}
+}
+
+func TestIsDBConnectionError(t *testing.T) {
+	tests := []struct {
+		err      error
+		name     string
+		expected bool
+	}{
+		{
+			name:     "nil",
+			err:      nil,
+			expected: false,
+		},
+		{
+			name:     "TCP connection refused",
+			err:      &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED},
+			expected: true,
+		},
+		{
+			name:     "TCP connection reset",
+			err:      &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET},
+			expected: true,
+		},
+		{
+			name:     "EOF",
+			err:      io.EOF,
+			expected: true,
+		},
+		{
+			name:     "unexpected EOF",
+			err:      io.ErrUnexpectedEOF,
+			expected: true,
+		},
+		{
+			name:     "wrapped TCP connection refused",
+			err:      fmt.Errorf("driver: %w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}),
+			expected: true,
+		},
+		{
+			name:     "database constraint violation",
+			err:      errors.New("violates unique constraint"),
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			RegisterTestingT(t)
+			Expect(IsDBConnectionError(tt.err)).To(Equal(tt.expected))
+		})
+	}
+}
diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go
index 6fad91b4..d9284665 100755
--- a/pkg/errors/errors.go
+++ b/pkg/errors/errors.go
@@ -395,6 +395,10 @@ func DatabaseAdvisoryLock(err error) *ServiceError {
 	return New(CodeInternalDatabase, "internal database error")
 }
 
+func ServiceUnavailable(reason string, values ...interface{}) *ServiceError {
+	return New(CodeServiceUnavailable, reason, values...)
+}
+
 func InvalidToken(reason string, values ...interface{}) *ServiceError {
 	return New(CodeAuthExpiredToken, reason, values...)
 }
diff --git a/pkg/errors/errors_test.go b/pkg/errors/errors_test.go
index ee167370..c361d0c0 100755
--- a/pkg/errors/errors_test.go
+++ b/pkg/errors/errors_test.go
@@ -128,6 +128,14 @@ func TestConstructors(t *testing.T) {
 			expectedType:   ErrorTypeInternal,
 			expectedReason: "internal database error",
 		},
+		{
+			name:           "ServiceUnavailable",
+			build:          func() *ServiceError { return ServiceUnavailable("database unreachable") },
+			expectedCode:   CodeServiceUnavailable,
+			expectedHTTP:   http.StatusServiceUnavailable,
+			expectedType:   ErrorTypeService,
+			expectedReason: "database unreachable",
+		},
 		{
 			name:           "InvalidToken",
 			build:          func() *ServiceError { return InvalidToken("token expired") },
diff --git a/pkg/services/generic.go b/pkg/services/generic.go
index b7c92ffe..73b24342 100755
--- a/pkg/services/generic.go
+++ b/pkg/services/generic.go
@@ -303,9 +303,12 @@ func (s *sqlGenericService) loadList(listCtx *listContext, d *dao.GenericDao) *e
 	// NOTE: Limit no longer supports '0' size and will cause issues. There is an early return, do not remove it.
 	//       https://github.com/go-gorm/gorm/blob/master/clause/limit.go#L18-L21
 	if err := (*d).Fetch((args.Page-1)*int(args.Size), int(args.Size), listCtx.resourceList); err != nil {
-		if e.Is(err, gorm.ErrRecordNotFound) {
+		switch {
+		case e.Is(err, gorm.ErrRecordNotFound):
 			listCtx.pagingMeta.Size = 0
-		} else {
+		case db.IsDBConnectionError(err):
+			return errors.ServiceUnavailable("Database connection unavailable")
+		default:
 			return errors.GeneralError("Unable to list resources: %s", err)
 		}
 	}
diff --git a/pkg/services/util.go b/pkg/services/util.go
index 0e2eb002..bb6738a8 100755
--- a/pkg/services/util.go
+++ b/pkg/services/util.go
@@ -6,6 +6,7 @@ import (
 
 	"gorm.io/gorm"
 
+	"github.com/openshift-hyperfleet/hyperfleet-api/pkg/db"
 	"github.com/openshift-hyperfleet/hyperfleet-api/pkg/errors"
 )
 
@@ -29,10 +30,16 @@ func handleGetError(resourceType, field string, value interface{}, err error) *e
 	if e.Is(err, gorm.ErrRecordNotFound) {
 		return errors.NotFound("%s with %s='%v' not found", resourceType, field, value)
 	}
+	if db.IsDBConnectionError(err) {
+		return errors.ServiceUnavailable("Database connection unavailable")
+	}
 	return errors.GeneralError("Unable to find %s with %s='%v': %s", resourceType, field, value, err)
 }
 
 func handleCreateError(resourceType string, err error) *errors.ServiceError {
+	if db.IsDBConnectionError(err) {
+		return errors.ServiceUnavailable("Database connection unavailable")
+	}
 	if strings.Contains(err.Error(), "violates unique constraint") {
 		return errors.Conflict("This %s already exists", resourceType)
 	}
@@ -40,6 +47,9 @@ func handleCreateError(resourceType string, err error) *errors.ServiceError {
 }
 
 func handleUpdateError(resourceType string, err error) *errors.ServiceError {
+	if db.IsDBConnectionError(err) {
+		return errors.ServiceUnavailable("Database connection unavailable")
+	}
 	if strings.Contains(err.Error(), "violates unique constraint") {
 		return errors.Conflict("Changes to %s conflict with existing records", resourceType)
 	}
@@ -47,5 +57,8 @@ func handleUpdateError(resourceType string, err error) *errors.ServiceError {
 }
 
 func handleDeleteError(resourceType string, err error) *errors.ServiceError {
+	if db.IsDBConnectionError(err) {
+		return errors.ServiceUnavailable("Database connection unavailable")
+	}
 	return errors.GeneralError("Unable to delete %s: %s", resourceType, err.Error())
 }
