mirror of
https://github.com/aljazceru/haven.git
synced 2025-12-18 22:24:22 +01:00
backup to s3, backup intervals
This commit is contained in:
Barry Deen
@@ -31,10 +31,11 @@ INBOX_PULL_INTERVAL_SECONDS=600
|
||||
## Import Settings
|
||||
IMPORT_START_DATE="2023-01-20"
|
||||
IMPORT_QUERY_INTERVAL_SECONDS=600
|
||||
IMPORT_SEED_RELAYS="relay.damus.io,nos.lol,relay.nostr.band,relay.snort.social,nostr.land,nostr.mom,relay.nos.social,relay.primal.net,relay.nostr.bg,no.str.cr,nostr21.com,nostrue.com,relay.siamstr.com"
|
||||
IMPORT_SEED_RELAYS="relay.damus.io,nos.lol,relay.nostr.band,relay.snort.social,nostr.land,nostr.mom,relay.nos.social,relay.primal.net,relay.nostr.bg,no.str.cr,nostr21.com,nostrue.com,relay.siamstr.com,wot.utxo.one,nostrelites.org,wot.nostr.party,wot.sovbit.host,wot.girino.org,relay.lnau.net,wot.siamstr.com,wot.sudocarlos.com,relay.otherstuff.fyi,relay.lexingtonbitcoin.org,wot.azzamo.net,wot.swarmstr.com,zap.watch,satsage.xyz,wons.calva.dev"
|
||||
|
||||
## Backup Settings
|
||||
BACKUP_PROVIDER="aws"
|
||||
BACKUP_INTERVAL_HOURS=24
|
||||
|
||||
## AWS Backup Settings - REQUIRED IF BACKUP_PROVIDER="aws"
|
||||
AWS_ACCESS_KEY_ID="AKIA"
|
||||
|
||||
24
README.md
24
README.md
@@ -170,6 +170,30 @@ Once everything is set up, the relay will be running on `localhost:3355` with th
|
||||
- `localhost:3355/chat`
|
||||
- `localhost:3355/inbox`
|
||||
|
||||
## Cloud Backups
|
||||
|
||||
The relay automatically backs up your database to a cloud provider of your choice.
|
||||
|
||||
### AWS
|
||||
|
||||
To back up your database to AWS, you'll need to first install and configure the awscli. You can do this by running the following commands:
|
||||
|
||||
```bash
|
||||
sudo python3 -m pip install awscli
|
||||
aws configure
|
||||
```
|
||||
|
||||
After configuring the awscli, you can set the following environment variables in your `.env` file:
|
||||
|
||||
```bash
|
||||
AWS_ACCESS_KEY_ID=your_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY=your_secret_access_key
|
||||
AWS_REGION=your_region
|
||||
AWS_BUCKET=your_bucket
|
||||
```
|
||||
|
||||
Replace `your_access_key_id`, `your_secret_access_key`, `your_region`, and `your_bucket` with your actual AWS credentials.
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the MIT License.
|
||||
|
||||
120
backup.go
Normal file
120
backup.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsConfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type AwsCredentials struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Region string
|
||||
Bucket string
|
||||
}
|
||||
|
||||
func backupDatabase() {
|
||||
ticker := time.NewTicker(time.Duration(config.BackupIntervalHours) * time.Hour)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
ZipDirectory("db", "db.zip")
|
||||
switch config.BackupProvider {
|
||||
case "aws":
|
||||
S3Upload()
|
||||
default:
|
||||
log.Println("🚫 we only support AWS at this time")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func S3Upload() {
|
||||
|
||||
bucket := getEnv("AWS_BUCKET_NAME")
|
||||
cfg, err := awsConfig.LoadDefaultConfig(context.TODO())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Create an Amazon S3 service client
|
||||
client := s3.NewFromConfig(cfg)
|
||||
|
||||
// Upload the file to S3
|
||||
file, err := os.Open("db.zip")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("db.zip"),
|
||||
Body: file,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("✅ Successfully uploaded %q to %q\n", "db.zip", bucket)
|
||||
|
||||
// delete the file
|
||||
err = os.Remove("db.zip")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func ZipDirectory(sourceDir, zipFileName string) error {
|
||||
log.Println("📦 zipping up the database")
|
||||
file, err := os.Create("db.zip")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
w := zip.NewWriter(file)
|
||||
defer w.Close()
|
||||
|
||||
walker := func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
f, err := w.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(f, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
err = filepath.Walk("db", walker)
|
||||
if err != nil {
|
||||
//panic(err)
|
||||
}
|
||||
|
||||
log.Println("📦 database zipped up!")
|
||||
return nil
|
||||
}
|
||||
@@ -37,6 +37,7 @@ type Config struct {
|
||||
ImportQueryIntervalSeconds int `json:"import_query_interval_seconds"`
|
||||
ImportSeedRelays []string `json:"import_seed_relays"`
|
||||
BackupProvider string `json:"backup_provider"`
|
||||
BackupIntervalHours int `json:"backup_interval_hours"`
|
||||
BlastrRelays []string `json:"blastr_relays"`
|
||||
}
|
||||
|
||||
@@ -78,6 +79,7 @@ func loadConfig() Config {
|
||||
ImportQueryIntervalSeconds: getEnvInt("IMPORT_QUERY_INTERVAL_SECONDS", 360000),
|
||||
ImportSeedRelays: getRelayList(getEnv("IMPORT_SEED_RELAYS")),
|
||||
BackupProvider: getEnv("BACKUP_PROVIDER"),
|
||||
BackupIntervalHours: getEnvInt("BACKUP_INTERVAL_HOURS", 24),
|
||||
BlastrRelays: getRelayList(getEnv("BLASTR_RELAYS")),
|
||||
}
|
||||
}
|
||||
|
||||
19
go.mod
19
go.mod
@@ -14,6 +14,24 @@ require (
|
||||
require (
|
||||
github.com/PowerDNS/lmdb-go v1.9.2 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.36 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.23.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.31.0 // indirect
|
||||
github.com/aws/smithy-go v1.21.0 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
|
||||
github.com/btcsuite/btcd/btcutil v1.1.3 // indirect
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
|
||||
@@ -33,6 +51,7 @@ require (
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/flatbuffers v23.5.26+incompatible // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.17.8 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
|
||||
40
go.sum
40
go.sum
@@ -5,6 +5,42 @@ github.com/PowerDNS/lmdb-go v1.9.2/go.mod h1:TE0l+EZK8Z1B4dx070ZxkWTlp8RG1mjN0/+
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U=
|
||||
github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.36 h1:4IlvHh6Olc7+61O1ktesh0jOcqmq/4WG6C2Aj5SKXy0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.36/go.mod h1:IiBpC0HPAGq9Le0Xxb1wpAKzEfAQ3XlYgJLYKEVYcfw=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.34 h1:gmkk1l/cDGSowPRzkdxYi8edw+gN4HmVK151D/pqGNc=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.34/go.mod h1:4R9OEV3tgFMsok4ZeFpExn7zQaZRa9MRGFYnI/xC/vs=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 h1:C/d03NAmh8C4BZXhuRNboF/DqhBkBCeDiJDcaqIT5pA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14/go.mod h1:7I0Ju7p9mCIdlrfS+JCgqcYD0VXz/N4yozsox+0o078=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18/go.mod h1:CUx0G1v3wG6l01tUB+j7Y8kclA8NSqK4ef0YG79a4cg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 h1:QFASJGfT8wMXtuP3D5CRmMjARHv9ZmzFUMJznHDOY3w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5/go.mod h1:QdZ3OmoIjSX+8D1OPAzPxDfjXASbBMDsz9qvtyIhtik=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 h1:Xbwbmk44URTiHNx6PNo0ujDE6ERlsCKJD3u1zfnzAPg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20/go.mod h1:oAfOFzUB14ltPZj1rWwRc3d/6OgD76R8KlvU3EqM9Fg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.0 h1:F6KG9CT7PPqAjnRxjKmYJopVnXPwjlzPI2FEgXHajNY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.0/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.23.0 h1:fHySkG0IGj2nepgGJPmmhZYL9ndnsq1Tvc6MeuVQCaQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.23.0/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.0 h1:cU/OeQPNReyMj1JEBgjE29aclYZYtXcsPMXbTkVGMFk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.0/go.mod h1:FnvDM4sfa+isJ3kDXIzAB9GAwVSzFzSy97uZ3IsHo4E=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.31.0 h1:GNVxIHBTi2EgwCxpNiozhNasMOK+ROUA2Z3X+cSBX58=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.31.0/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI=
|
||||
github.com/aws/smithy-go v1.21.0 h1:H7L8dtDRk0P1Qm6y0ji7MCYMQObJ5R9CRpyPhRUkLYA=
|
||||
github.com/aws/smithy-go v1.21.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
|
||||
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
|
||||
github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=
|
||||
@@ -112,6 +148,9 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
@@ -268,6 +307,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
Reference in New Issue
Block a user