diff --git a/.env.example b/.env.example index 95cf2eb..c3de8ac 100644 --- a/.env.example +++ b/.env.example @@ -31,10 +31,11 @@ INBOX_PULL_INTERVAL_SECONDS=600 ## Import Settings IMPORT_START_DATE="2023-01-20" IMPORT_QUERY_INTERVAL_SECONDS=600 -IMPORT_SEED_RELAYS="relay.damus.io,nos.lol,relay.nostr.band,relay.snort.social,nostr.land,nostr.mom,relay.nos.social,relay.primal.net,relay.nostr.bg,no.str.cr,nostr21.com,nostrue.com,relay.siamstr.com" +IMPORT_SEED_RELAYS="relay.damus.io,nos.lol,relay.nostr.band,relay.snort.social,nostr.land,nostr.mom,relay.nos.social,relay.primal.net,relay.nostr.bg,no.str.cr,nostr21.com,nostrue.com,relay.siamstr.com,wot.utxo.one,nostrelites.org,wot.nostr.party,wot.sovbit.host,wot.girino.org,relay.lnau.net,wot.siamstr.com,wot.sudocarlos.com,relay.otherstuff.fyi,relay.lexingtonbitcoin.org,wot.azzamo.net,wot.swarmstr.com,zap.watch,satsage.xyz,wons.calva.dev" ## Backup Settings BACKUP_PROVIDER="aws" +BACKUP_INTERVAL_HOURS=24 ## AWS Backup Settings - REQUIRED IF BACKUP_PROVIDER="aws" AWS_ACCESS_KEY_ID="AKIA" diff --git a/README.md b/README.md index ab15409..c543bee 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,30 @@ Once everything is set up, the relay will be running on `localhost:3355` with th - `localhost:3355/chat` - `localhost:3355/inbox` +## Cloud Backups + +The relay automatically backs up your database to a cloud provider of your choice. + +### AWS + +To back up your database to AWS, you'll need to first install and configure the awscli. You can do this by running the following commands: + +```bash +sudo python3 -m pip install awscli +aws configure +``` + +After configuring the awscli, you can set the following environment variables in your `.env` file: + +```bash +AWS_ACCESS_KEY_ID=your_access_key_id +AWS_SECRET_ACCESS_KEY=your_secret_access_key +AWS_REGION=your_region +AWS_BUCKET=your_bucket +``` + +Replace `your_access_key_id`, `your_secret_access_key`, `your_region`, and `your_bucket` with your actual AWS credentials. + ## License This project is licensed under the MIT License. diff --git a/backup.go b/backup.go new file mode 100644 index 0000000..9944203 --- /dev/null +++ b/backup.go @@ -0,0 +1,120 @@ +package main + +import ( + "archive/zip" + "context" + "io" + "log" + "os" + "path/filepath" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type AwsCredentials struct { + AccessKeyID string + SecretAccessKey string + Region string + Bucket string +} + +func backupDatabase() { + ticker := time.NewTicker(time.Duration(config.BackupIntervalHours) * time.Hour) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + ZipDirectory("db", "db.zip") + switch config.BackupProvider { + case "aws": + S3Upload() + default: + log.Println("🚫 we only support AWS at this time") + } + } + } +} + +func S3Upload() { + + bucket := getEnv("AWS_BUCKET_NAME") + cfg, err := awsConfig.LoadDefaultConfig(context.TODO()) + if err != nil { + log.Fatal(err) + } + + // Create an Amazon S3 service client + client := s3.NewFromConfig(cfg) + + // Upload the file to S3 + file, err := os.Open("db.zip") + if err != nil { + log.Fatal(err) + } + + _, err = client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String("db.zip"), + Body: file, + }) + + if err != nil { + log.Fatal(err) + } + + log.Printf("✅ Successfully uploaded %q to %q\n", "db.zip", bucket) + + // delete the file + err = os.Remove("db.zip") + if err != nil { + log.Fatal(err) + } +} + +func ZipDirectory(sourceDir, zipFileName string) error { + log.Println("📦 zipping up the database") + file, err := os.Create("db.zip") + if err != nil { + panic(err) + } + defer file.Close() + + w := zip.NewWriter(file) + defer w.Close() + + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + f, err := w.Create(path) + if err != nil { + return err + } + + _, err = io.Copy(f, file) + if err != nil { + return err + } + + return nil + } + err = filepath.Walk("db", walker) + if err != nil { + //panic(err) + } + + log.Println("📦 database zipped up!") + return nil +} diff --git a/config.go b/config.go index 14374a5..8288732 100644 --- a/config.go +++ b/config.go @@ -37,6 +37,7 @@ type Config struct { ImportQueryIntervalSeconds int `json:"import_query_interval_seconds"` ImportSeedRelays []string `json:"import_seed_relays"` BackupProvider string `json:"backup_provider"` + BackupIntervalHours int `json:"backup_interval_hours"` BlastrRelays []string `json:"blastr_relays"` } @@ -78,6 +79,7 @@ func loadConfig() Config { ImportQueryIntervalSeconds: getEnvInt("IMPORT_QUERY_INTERVAL_SECONDS", 360000), ImportSeedRelays: getRelayList(getEnv("IMPORT_SEED_RELAYS")), BackupProvider: getEnv("BACKUP_PROVIDER"), + BackupIntervalHours: getEnvInt("BACKUP_INTERVAL_HOURS", 24), BlastrRelays: getRelayList(getEnv("BLASTR_RELAYS")), } } diff --git a/go.mod b/go.mod index d82a201..9809921 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,24 @@ require ( require ( github.com/PowerDNS/lmdb-go v1.9.2 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.36 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.34 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.63.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.23.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.31.0 // indirect + github.com/aws/smithy-go v1.21.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/btcsuite/btcd/btcutil v1.1.3 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect @@ -33,6 +51,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 8a08d60..da8df5f 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,42 @@ github.com/PowerDNS/lmdb-go v1.9.2/go.mod h1:TE0l+EZK8Z1B4dx070ZxkWTlp8RG1mjN0/+ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U= +github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc= +github.com/aws/aws-sdk-go-v2/config v1.27.36 h1:4IlvHh6Olc7+61O1ktesh0jOcqmq/4WG6C2Aj5SKXy0= +github.com/aws/aws-sdk-go-v2/config v1.27.36/go.mod h1:IiBpC0HPAGq9Le0Xxb1wpAKzEfAQ3XlYgJLYKEVYcfw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.34 h1:gmkk1l/cDGSowPRzkdxYi8edw+gN4HmVK151D/pqGNc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.34/go.mod h1:4R9OEV3tgFMsok4ZeFpExn7zQaZRa9MRGFYnI/xC/vs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 h1:C/d03NAmh8C4BZXhuRNboF/DqhBkBCeDiJDcaqIT5pA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14/go.mod h1:7I0Ju7p9mCIdlrfS+JCgqcYD0VXz/N4yozsox+0o078= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18/go.mod h1:CUx0G1v3wG6l01tUB+j7Y8kclA8NSqK4ef0YG79a4cg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 h1:QFASJGfT8wMXtuP3D5CRmMjARHv9ZmzFUMJznHDOY3w= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5/go.mod h1:QdZ3OmoIjSX+8D1OPAzPxDfjXASbBMDsz9qvtyIhtik= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 h1:Xbwbmk44URTiHNx6PNo0ujDE6ERlsCKJD3u1zfnzAPg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20/go.mod h1:oAfOFzUB14ltPZj1rWwRc3d/6OgD76R8KlvU3EqM9Fg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.63.0 h1:F6KG9CT7PPqAjnRxjKmYJopVnXPwjlzPI2FEgXHajNY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.63.0/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.23.0 h1:fHySkG0IGj2nepgGJPmmhZYL9ndnsq1Tvc6MeuVQCaQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.23.0/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.0 h1:cU/OeQPNReyMj1JEBgjE29aclYZYtXcsPMXbTkVGMFk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.0/go.mod h1:FnvDM4sfa+isJ3kDXIzAB9GAwVSzFzSy97uZ3IsHo4E= +github.com/aws/aws-sdk-go-v2/service/sts v1.31.0 h1:GNVxIHBTi2EgwCxpNiozhNasMOK+ROUA2Z3X+cSBX58= +github.com/aws/aws-sdk-go-v2/service/sts v1.31.0/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI= +github.com/aws/smithy-go v1.21.0 h1:H7L8dtDRk0P1Qm6y0ji7MCYMQObJ5R9CRpyPhRUkLYA= +github.com/aws/smithy-go v1.21.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= @@ -112,6 +148,9 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -268,6 +307,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index 5523e38..1d3935d 100644 --- a/main.go +++ b/main.go @@ -38,6 +38,7 @@ func main() { go refreshTrustNetwork() go subscribeInbox() + go backupDatabase() handler := http.HandlerFunc(dynamicRelayHandler)