syntax

Added exclusion of hidden files

jenkins/backupDavid.yml

@@ -0,0 +1,12 @@
+pipeline {
+    agent any
+    stages {
+        stage ('Run Backup to David') {
+            steps {
+                script {
+                    sh "rsync -v -a -e 'ssh -p 664' --delete --bwlimit=3000 --info=progress2 ${env.borglocation} matthias@home.daf2000.nl:/media/disk/"
+                }
+            }
+        }
+    }
+}

jenkins/backupWinschoten.yml

@@ -0,0 +1,14 @@
+pipeline {
+    parameters {
+    agent any
+    stages {
+        stage ('Run Backup to David') {
+            steps {
+                script {
+                    sh "cp ${keyfile} ${WORKSPACE}/keyfile"
+                    sh "rsync -v -a -e 'ssh -i /home/backupwinschoten/.ssh/id_rsa -p 9897' --delete --info=progress2 ${env.borglocation} backupwinschoten@localhost:/storage/"
+                }
+            }
+        }
+    }
+}

jenkins/default_backup.yml

@@ -1,85 +1,26 @@
 pipeline {
-    agent any
     parameters {
-        booleanParam(name: 'backuplocally', defaultValue: false, description: 'If true local BORG backup will be created')
-        booleanParam(name: 'backuptostackstorage', defaultValue: false, description: 'If true backup data to TransIP Stack')
-        booleanParam(name: 'backuptodavid', defaultValue: false, description: 'If true rsync the BORG repository to David')
-        booleanParam(name: 'backuptopi', defaultValue: false, description: 'If true rsync the BORG repository to our ExternalPI')
+        string(name: 'directory', defaultValue: params.directory ?: '${env.borglocation}/', description: 'The directory that should be handled')
+        string(name: 'agentname', defaultValue: params.agentname ?: 'julien', description: 'The agent to run this backup on')
+        text(name: 'excludelist', defaultValue:  params.excludelist ?: '**/cache/** ', description: 'Multiline string to exclude patterns from backup')
+    }
+    agent { label params['agentname'] }
+    environment {
+        BORG_RELOCATED_REPO_ACCESS_IS_OK = 'yes'
     }
     stages {
       stage('Run Information') {
             steps {
-                echo "Local BORG backup creation is ${params.backuplocally}"
-                echo "Backup to TransIP Stack is ${params.backuptostackstorage}"
-                echo "Backup to David is ${params.backuptodavid}"
-                echo "Backup to External PI is ${params.backuptopi}"
-                echo "Excludelist: ${params.excludelist}"
+                echo "Backing up directory ${directory}"
                 sh "echo '${params.excludelist}' >> excludelist"
             }
         }
-        stage('Getting RClone Configuration') {
-            steps {
-                configFileProvider([configFile(fileId: 'e0237193-1245-452e-b035-9d3d501f4c1b', variable: 'rclone_config')]) {
-                sh "mkdir -p ${WORKSPACE}/config"
-                sh "cp ${rclone_config} ${WORKSPACE}/config/rclone.conf"
-                }
-            }
-        }
         stage('Create local BorgBackup'){
             steps {
                 script {
-                    if (params.backuplocally) {
                     withCredentials([string(credentialsId: 'cbce976a-0d98-4f35-8ea2-1f7818931bc3', variable: 'BORG_PASSPHRASE')]) {
-                          sh "borg create --progress --stats --exclude-from excludelist ${env.borglocation}/${directory}::${java.time.LocalDateTime.now().format(java.time.format.DateTimeFormatter.ofPattern('dd-MM-yyyy_HH:mm'))} ${env.storagelocation}/${directory}"
-                          sh "borg prune --list --keep-daily 31 --keep-weekly 48 ${env.borglocation}/${directory}"
-                        }
-                    } else {
-                        echo "Local BORG backup creation is skipped"
-                    }
-                }
-            }
-        }
-       stage('Run rclone') {
-            agent {
-                docker {
-                    image 'rclone/rclone'
-                    args "--volumes-from=jenkins -v ${env.storagelocation}/${directory}/:/data/ --entrypoint=''"
-                    reuseNode true
-                }
-            }
-            steps {
-                script {
-                    if (params.backuptostackstorage) {
-                        sh "mkdir -p /config/rclone"
-                        sh "cp ${WORKSPACE}/config/rclone.conf /config/rclone/"
-                        sh "rclone copy -v /data/ stackstorage:/julien/storage/${directory}"
-                    } else {
-                        echo "Backup to TransIP Stack is skipped"
-                    }
-                }
-            }
-        }
-        stage('Run Rsync to David') {
-            steps {
-                script {
-                    if (params.backuptodavid) {
-                        sh "rsync -v -a -e 'ssh -p 664' --delete --bwlimit=3000 --info=progress2 ${env.borglocation}/${directory} matthias@home.daf2000.nl:/media/disk/borgbackup/${directory}"
-                    } else {
-                        echo "Backup to David is skipped"
-                    }
-                }
-            }
-        }
-        stage('Run Rsync to External PI') {
-            steps {
-                script {
-                    if (params.backuptopi) {
-                        withCredentials([sshUserPrivateKey(credentialsId: '095cc365-ac40-4ddb-a078-2fa403092de0', keyFileVariable: 'keyfile', passphraseVariable: 'passphrase', usernameVariable: 'user')]) {
-                            sh "cp ${keyfile} ${WORKSPACE}/keyfile"
-                            sh "rsync -v -a -e 'ssh -i /home/borgbackup/.ssh/id_rsa -p 9898' --delete --info=progress2 ${env.borglocation}/${directory} borgbackup@localhost:/borgbackup/${directory}"
-                        }
-                    } else {
-                        echo "Backup to External PI is skipped"
+                      sh "borg create --progress --stats --exclude-from excludelist ${directory}::${java.time.LocalDateTime.now().format(java.time.format.DateTimeFormatter.ofPattern('dd-MM-yyyy_HH:mm'))} ${env.storagelocation}/${directory}"
+                      sh "borg prune --list --keep-daily 31 --keep-weekly 48 ${directory}"
                     }
                 }
             }

jenkins/nextcloud_database.yml

@@ -0,0 +1,95 @@
+pipeline {
+    agent any
+    parameters {
+        booleanParam(name: 'backuplocally', defaultValue:  params.backuplocally ?: false, description: 'If true local BORG backup will be created')
+        booleanParam(name: 'backuptodavid', defaultValue: params.backuptodavid ?: false, description: 'If true rsync the BORG repository to David')
+        booleanParam(name: 'backuptowinschoten', defaultValue: params.backuptowinschoten ?: false, description: 'If true rsync the BORG repository to our Winschoten backup system')
+    }
+    
+    stages {
+      stage('Run Information') {
+            steps {
+                echo "Local BORG backup creation is ${params.backuplocally}"
+                echo "Backup to David is ${params.backuptodavid}"
+                echo "Backup to Winschoten PI is ${params.backuptowinschoten}"
+            }
+        }
+        stage('Run MariaDB') {
+            agent {
+                docker {
+                    image 'mariadb:latest'
+                    args "--volumes-from=jenkins -v /srv/dev-disk-by-uuid-27fc012e-a1fa-4c7c-9dad-82770888cd03/nextcloud_backup/database_dump:/data/ --entrypoint=''"
+                    reuseNode false
+                }
+            }
+            steps {
+                script {
+                    if (params.backuplocally) {
+                        sh "yes | cp -rf /data/backup_yesterday.dmp /data/backup_early.dmp 2>/dev/null || :"
+                        sh "yes | cp -rf /data/backup_latest.dmp /data/backup_yesterday.dmp 2>/dev/null || :"
+                        sh "mariadb-dump -u nextcloud -h 10.10.1.32 --all-databases -ppassword -v > /data/backup_latest.dmp"
+                    } else {
+                        echo "Local backup creation is skipped"
+                    }
+                }
+            }
+        }
+        stage('Getting RClone Configuration') {
+            steps {
+                configFileProvider([configFile(fileId: 'e0237193-1245-452e-b035-9d3d501f4c1b', variable: 'rclone_config')]) {
+                sh "mkdir -p ${WORKSPACE}/config"
+                sh "cp ${rclone_config} ${WORKSPACE}/config/rclone.conf"
+                }
+            }
+        }
+
+        stage('Rsync Backup in Parallel') {
+            failFast false
+            parallel {
+                stage ('Run Backup to David') {
+                    steps {
+                        script {
+                            if (params.backuptodavid) {
+                                sh "yes | cp -rf matthias@home.daf2000.nl:/media/disk/nextcloudbackup/database_dump/backup_yesterday.dmp matthias@home.daf2000.nl:/media/disk/nextcloudbackup/database_dump/backup_early.dmp 2>/dev/null || :"
+                                sh "yes | cp -rf matthias@home.daf2000.nl:/media/disk/nextcloudbackup/database_dump/backup_latest.dmp matthias@home.daf2000.nl:/media/disk/nextcloudbackup/database_dump/backup_yesterday.dmp 2>/dev/null || :"
+                                sh "rsync -v -a -e 'ssh -p 664' --delete --bwlimit=3000 --info=progress2 --exclude='.*' /srv/dev-disk-by-uuid-27fc012e-a1fa-4c7c-9dad-82770888cd03/nextcloud_backup/database_dump/ matthias@home.daf2000.nl:/media/disk/nextcloudbackup/database_dump/"
+                            } else {
+                                echo "Backup to David is skipped"
+                            }
+                        }
+                    }
+                }
+                stage ('Run Backup to Winschoten') {
+                    steps {
+                        script {
+                            if (params.backuptowinschoten) {
+                                withCredentials([sshUserPrivateKey(credentialsId: 'backupwinschoten', keyFileVariable: 'keyfile', passphraseVariable: 'passphrase', usernameVariable: 'user')]) {
+                                    sh "cp ${keyfile} ${WORKSPACE}/keyfile"
+                                    sh "rsync -v -a -e 'ssh -i /home/backupwinschoten/.ssh/id_rsa -p 9897' --delete --info=progress2 ${env.borglocation}/${directory} backupwinschoten@localhost:/storage/borgbackups/"
+                                }
+                            } else {
+                                echo "Backup to Winschoten is skipped"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        
+        stage('Run Rsync to External PI') {
+            steps {
+                script {
+                    if (params.backuptopi) {
+                        withCredentials([sshUserPrivateKey(credentialsId: '095cc365-ac40-4ddb-a078-2fa403092de0', keyFileVariable: 'keyfile', passphraseVariable: 'passphrase', usernameVariable: 'user')]) {
+                            sh "cp ${keyfile} ${WORKSPACE}/keyfile"
+                            sh "rsync -v -a -e 'ssh -i /home/borgbackup/.ssh/id_rsa -p 9898' --delete --info=progress2 --exclude='.*' /srv/dev-disk-by-uuid-27fc012e-a1fa-4c7c-9dad-82770888cd03/nextcloud_backup/database_dump/ borgbackup@localhost:/nextcloudbackup/database_dump/"
+                        }
+                    } else {
+                        echo "Backup to External PI is skipped"
+                    }
+                }
+            }
+        }
+    }
+}