From 35720c2422ddcddfd4d5f56cd1d5712cd9a4725d Mon Sep 17 00:00:00 2001 From: thomas-tacquet Date: Tue, 12 Nov 2024 16:14:15 +0100 Subject: [PATCH 1/2] feat(jobs): add database backup example --- jobs/databases-backup/Dockerfile | 14 ++++ jobs/databases-backup/go.mod | 7 ++ jobs/databases-backup/go.sum | 6 ++ jobs/databases-backup/main.go | 133 +++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 jobs/databases-backup/Dockerfile create mode 100644 jobs/databases-backup/go.mod create mode 100644 jobs/databases-backup/go.sum create mode 100644 jobs/databases-backup/main.go diff --git a/jobs/databases-backup/Dockerfile b/jobs/databases-backup/Dockerfile new file mode 100644 index 0000000..41c4fd2 --- /dev/null +++ b/jobs/databases-backup/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.23-alpine + +# Set destination for COPY +WORKDIR /app + +COPY go.mod ./ +COPY go.sum ./ +COPY *.go ./ + +# Build +RUN go build -o /server-image + +# Run +CMD [ "/server-image" ] diff --git a/jobs/databases-backup/go.mod b/jobs/databases-backup/go.mod new file mode 100644 index 0000000..1566558 --- /dev/null +++ b/jobs/databases-backup/go.mod @@ -0,0 +1,7 @@ +module gitlab.infra.online.net/ttacquet/jobs-demo + +go 1.23 + +require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 + +require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/jobs/databases-backup/go.sum b/jobs/databases-backup/go.sum new file mode 100644 index 0000000..94acdfa --- /dev/null +++ b/jobs/databases-backup/go.sum @@ -0,0 +1,6 @@ +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/jobs/databases-backup/main.go b/jobs/databases-backup/main.go new file mode 100644 index 0000000..dfeec83 --- /dev/null +++ b/jobs/databases-backup/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "time" + + "github.com/scaleway/scaleway-sdk-go/api/rdb/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +const ( + // Defining variable or secret readings. + VAR_ORG_ID = "SCW_DEFAULT_ORGANIZATION_ID" + VAR_AK = "SCW_ACCESS_KEY" + VAR_SK = "SCW_SECRET_KEY" + VAR_REGION = "REGION" + VAR_RDB_ID = "INSTANCE_ID" + + // optional, never expires if not definied. + VAR_EXPIRE_AT_DAYS = "EXPIRE_AT_DAYS" +) + +func main() { + fmt.Println("creating backup of managed database...") + + // Create a Scaleway client with credentials from environment variables. + client, err := scw.NewClient( + // Get your organization ID at https://console.scaleway.com/organization/settings + scw.WithDefaultOrganizationID(os.Getenv(VAR_ORG_ID)), + + // Get your credentials at https://console.scaleway.com/iam/api-keys + scw.WithAuth(os.Getenv(VAR_AK), os.Getenv(VAR_SK)), + + // Get more about our availability zones at https://www.scaleway.com/en/docs/console/my-account/reference-content/products-availability/ + scw.WithDefaultRegion(scw.RegionFrPar), + ) + if err != nil { + panic(err) + } + + rdbAPI := rdb.NewAPI(client) + + if err := createRdbSnapshot(rdbAPI); err != nil { + panic(err) + } +} + +func createRdbSnapshot(rdbAPI *rdb.API) error { + rdbInstance, err := rdbAPI.GetInstance(&rdb.GetInstanceRequest{ + Region: scw.Region(os.Getenv(VAR_REGION)), + InstanceID: os.Getenv(VAR_RDB_ID), + }) + if err != nil { + return fmt.Errorf("error while getting database instance %w", err) + } + + databasesList, err := rdbAPI.ListDatabases(&rdb.ListDatabasesRequest{ + Region: scw.Region(os.Getenv(VAR_REGION)), + InstanceID: rdbInstance.ID, + }) + if err != nil { + return fmt.Errorf("error while listing databases %w", err) + } + + expiresAt, err := getExpirationDate() + if err != nil { + return fmt.Errorf("error while getting expiration date %w", err) + } + + tn := time.Now() + backupName := fmt.Sprintf("backup_%s_%d%d%d", rdbInstance.Name, tn.Year(), tn.Month(), tn.Day()) + + for _, database := range databasesList.Databases { + + backup, err := rdbAPI.CreateDatabaseBackup(&rdb.CreateDatabaseBackupRequest{ + Region: scw.Region(os.Getenv(VAR_REGION)), + InstanceID: rdbInstance.ID, + Name: backupName, + DatabaseName: database.Name, + ExpiresAt: expiresAt, + }) + if err != nil { + return fmt.Errorf("error while creating database backup request %w", err) + } + + fmt.Println("Created backup ", backup.Name) + } + + return nil +} + +func getExpirationDate() (*time.Time, error) { + var expiresAt *time.Time + expireDays := os.Getenv(VAR_EXPIRE_AT_DAYS) + + if expireDays != "" { + expireDaysInt, err := strconv.Atoi(expireDays) + if err != nil { + return nil, fmt.Errorf("error while getting %w", err) + } + + if expireDaysInt > 0 { + expiration := time.Now().AddDate(0, 0, expireDaysInt) + expiresAt = &expiration + } + } + + return expiresAt, nil +} + +func init() { + if os.Getenv(VAR_ORG_ID) == "" { + panic("missing " + VAR_ORG_ID) + } + + if os.Getenv(VAR_AK) == "" { + panic("missing " + VAR_AK) + } + + if os.Getenv(VAR_SK) == "" { + panic("missing " + VAR_SK) + } + + if os.Getenv(VAR_RDB_ID) == "" { + panic("missing " + VAR_RDB_ID) + } + + if os.Getenv(VAR_REGION) == "" { + panic("missing " + VAR_REGION) + } +} From c7f0dc28c26787cf778fba59e741f27b94e1e39d Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Fri, 21 Mar 2025 16:32:09 +0100 Subject: [PATCH 2/2] update in progress --- jobs/databases-backup/Dockerfile | 2 +- jobs/databases-backup/go.mod | 4 +- jobs/databases-backup/go.sum | 4 +- jobs/databases-backup/main.go | 81 +++++++++++++++----------------- 4 files changed, 43 insertions(+), 48 deletions(-) diff --git a/jobs/databases-backup/Dockerfile b/jobs/databases-backup/Dockerfile index 41c4fd2..11aaed4 100644 --- a/jobs/databases-backup/Dockerfile +++ b/jobs/databases-backup/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine +FROM golang:1.24.1-bookworm # Set destination for COPY WORKDIR /app diff --git a/jobs/databases-backup/go.mod b/jobs/databases-backup/go.mod index 1566558..473a401 100644 --- a/jobs/databases-backup/go.mod +++ b/jobs/databases-backup/go.mod @@ -1,7 +1,7 @@ module gitlab.infra.online.net/ttacquet/jobs-demo -go 1.23 +go 1.24 -require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 +require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32 require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/jobs/databases-backup/go.sum b/jobs/databases-backup/go.sum index 94acdfa..1eb87cc 100644 --- a/jobs/databases-backup/go.sum +++ b/jobs/databases-backup/go.sum @@ -1,5 +1,5 @@ -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32 h1:4+LP7qmsLSGbmc66m1s5dKRMBwztRppfxFKlYqYte/c= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.32/go.mod h1:kzh+BSAvpoyHHdHBCDhmSWtBc1NbLMZ2lWHqnBoxFks= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/jobs/databases-backup/main.go b/jobs/databases-backup/main.go index dfeec83..8b5f7d3 100644 --- a/jobs/databases-backup/main.go +++ b/jobs/databases-backup/main.go @@ -11,30 +11,44 @@ import ( ) const ( - // Defining variable or secret readings. - VAR_ORG_ID = "SCW_DEFAULT_ORGANIZATION_ID" - VAR_AK = "SCW_ACCESS_KEY" - VAR_SK = "SCW_SECRET_KEY" - VAR_REGION = "REGION" - VAR_RDB_ID = "INSTANCE_ID" - - // optional, never expires if not definied. - VAR_EXPIRE_AT_DAYS = "EXPIRE_AT_DAYS" + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" // Scaleway organization ID + envAccessKey = "SCW_ACCESS_KEY" // Scaleway API access key + envSecretKey = "SCW_SECRET_KEY" // Scaleway API secret key + envProjectID = "SCW_PROJECT_ID" // Scaleway project ID + + envRegion = "SCW_REGION" + envDatabaseID = "SCW_RDB_ID" + envBackupExpirationDays = "SCW_EXPIRATION_DAYS" ) +// Check for mandatory variables before starting to work. +func init() { + // Slice of environmental variables that must be set for the application to run + mandatoryVariables := [...]string{envOrgID, envAccessKey, envSecretKey, envProjectID, envRegion} + + // Iterate through the slice and check if any variables are not set + for idx := range mandatoryVariables { + if os.Getenv(mandatoryVariables[idx]) == "" { + panic("missing environment variable " + mandatoryVariables[idx]) + } + } +} + func main() { fmt.Println("creating backup of managed database...") - // Create a Scaleway client with credentials from environment variables. + // Create a Scaleway client with credentials provided via environment variables. + // The client is used to interact with the Scaleway API client, err := scw.NewClient( // Get your organization ID at https://console.scaleway.com/organization/settings - scw.WithDefaultOrganizationID(os.Getenv(VAR_ORG_ID)), + scw.WithDefaultOrganizationID(os.Getenv(envOrgID)), // Get your credentials at https://console.scaleway.com/iam/api-keys - scw.WithAuth(os.Getenv(VAR_AK), os.Getenv(VAR_SK)), + scw.WithAuth(os.Getenv(envAccessKey), os.Getenv(envSecretKey)), - // Get more about our availability zones at https://www.scaleway.com/en/docs/console/my-account/reference-content/products-availability/ - scw.WithDefaultRegion(scw.RegionFrPar), + // Get more about our availability + // zones at https://www.scaleway.com/en/docs/console/my-account/reference-content/products-availability/ + scw.WithDefaultRegion(scw.Region(os.Getenv(envRegion))), ) if err != nil { panic(err) @@ -49,15 +63,15 @@ func main() { func createRdbSnapshot(rdbAPI *rdb.API) error { rdbInstance, err := rdbAPI.GetInstance(&rdb.GetInstanceRequest{ - Region: scw.Region(os.Getenv(VAR_REGION)), - InstanceID: os.Getenv(VAR_RDB_ID), + Region: scw.Region(scw.Region(os.Getenv(envRegion))), + InstanceID: os.Getenv(envDatabaseID), }) if err != nil { return fmt.Errorf("error while getting database instance %w", err) } databasesList, err := rdbAPI.ListDatabases(&rdb.ListDatabasesRequest{ - Region: scw.Region(os.Getenv(VAR_REGION)), + Region: scw.Region(os.Getenv(envRegion)), InstanceID: rdbInstance.ID, }) if err != nil { @@ -69,13 +83,16 @@ func createRdbSnapshot(rdbAPI *rdb.API) error { return fmt.Errorf("error while getting expiration date %w", err) } - tn := time.Now() - backupName := fmt.Sprintf("backup_%s_%d%d%d", rdbInstance.Name, tn.Year(), tn.Month(), tn.Day()) + now := time.Now() for _, database := range databasesList.Databases { + backupName := fmt.Sprintf("backup-%s-%s-%s", + database.Name, + now, + os.Getenv(envRegion)) backup, err := rdbAPI.CreateDatabaseBackup(&rdb.CreateDatabaseBackupRequest{ - Region: scw.Region(os.Getenv(VAR_REGION)), + Region: scw.Region(os.Getenv(envRegion)), InstanceID: rdbInstance.ID, Name: backupName, DatabaseName: database.Name, @@ -93,7 +110,7 @@ func createRdbSnapshot(rdbAPI *rdb.API) error { func getExpirationDate() (*time.Time, error) { var expiresAt *time.Time - expireDays := os.Getenv(VAR_EXPIRE_AT_DAYS) + expireDays := os.Getenv(envBackupExpirationDays) if expireDays != "" { expireDaysInt, err := strconv.Atoi(expireDays) @@ -109,25 +126,3 @@ func getExpirationDate() (*time.Time, error) { return expiresAt, nil } - -func init() { - if os.Getenv(VAR_ORG_ID) == "" { - panic("missing " + VAR_ORG_ID) - } - - if os.Getenv(VAR_AK) == "" { - panic("missing " + VAR_AK) - } - - if os.Getenv(VAR_SK) == "" { - panic("missing " + VAR_SK) - } - - if os.Getenv(VAR_RDB_ID) == "" { - panic("missing " + VAR_RDB_ID) - } - - if os.Getenv(VAR_REGION) == "" { - panic("missing " + VAR_REGION) - } -}