From f45e91f7808e1795c17fedc93b44cffe2b741bb9 Mon Sep 17 00:00:00 2001
From: Kilian Saffran <ksaffran@dks.lu>
Date: Fri, 5 Jul 2019 18:23:07 +0200
Subject: [PATCH] auto commit on 2019-07-05 18:23

---
 bin/db_backup.pl   |  71 ++++++++++++++++++++
 bin/db_backup.sh   |  40 -----------
 bin/file_backup.pl | 162 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 233 insertions(+), 40 deletions(-)
 create mode 100644 bin/db_backup.pl
 delete mode 100644 bin/db_backup.sh
 create mode 100644 bin/file_backup.pl

diff --git a/bin/db_backup.pl b/bin/db_backup.pl
new file mode 100644
index 0000000..d6ef32b
--- /dev/null
+++ b/bin/db_backup.pl
@@ -0,0 +1,71 @@
+#!/usr/bin/perl
+
+use strict;
+use Getopt::Long;
+use POSIX qw/strftime/;
+use JSON::PP;
+#use Data::Dumper;
+my $strdbs = ""; 
+GetOptions("db=s" => \$strdbs);
+my $cfgpath="/home/dks/backup/webserver/conf";
+my $logfile="/home/dks/backup/webserver/log/".strftime('%Y%m',localtime()).'.log';
+
+opendir(CGD,$cfgpath);
+my $dbcfg = ();
+while (my $f = readdir(CGD)){
+  if ($f !~ /\.json$/){ next; }
+  my $file = $f;
+  $file =~ s/\.db\.json$//;
+  if ($strdbs eq "" || $file =~ /^$strdbs/){
+    my $strcfg = "";
+    open(CFG,$cfgpath.'/'.$f);
+    while (my $l = <CFG>){
+      $strcfg .= $l; 
+    }
+    close(CFG);
+    $dbcfg->{$file} = JSON::PP::decode_json($strcfg);
+  }
+   
+}
+closedir(CGD);
+#print Dumper($dbcfg);
+open(LOG,">>".$logfile);
+foreach my $db (keys(%{$dbcfg})){
+  print LOG strftime("==\n%Y-%m-%d %H:%M",localtime()).": backup DB : $db\n";
+  my @cmd = ();
+  if (! -d $dbcfg->{$db}->{'bkppath'}){
+    mkdir($dbcfg->{$db}->{'bkppath'});
+  }
+  my $outfile = $dbcfg->{$db}->{'bkppath'}.'/'.$db.'_'.strftime("%Y%m%d_%H%M%S",localtime()).'.sql';
+  if ($dbcfg->{$db}->{type} eq 'pgsql'){
+    foreach my $c (keys(%{$dbcfg->{$db}})){
+      if ($c =~ /^PGPASSWORD/){
+        push(@cmd,"export ".$c.'='.$dbcfg->{$db}->{$c});
+      }
+    }
+  }
+  
+  if ($dbcfg->{$db}->{type} eq "mysql"){
+    push(@cmd,'mysqldump -h '.$dbcfg->{$db}->{'dbhost'}.' -u '.$dbcfg->{$db}->{'dbuser'}.' -p'.$dbcfg->{$db}->{'dbpwd'}.' '.$dbcfg->{$db}->{'dbname'}.' > '.$outfile);
+  } elsif ($dbcfg->{$db}->{type} eq "pgsql"){
+    push(@cmd,'pg_dump -h '.$dbcfg->{$db}->{PGHOST}.' -U '.$dbcfg->{$db}->{PGUSER}.' -F p -O -x -w '.$dbcfg->{$db}->{'dbname'}.' > '.$outfile);
+  }
+  push (@cmd,'gzip '.$outfile);
+  my $cmd1 = join(" && ",@cmd);
+  my $ret = `$cmd1`;
+  print LOG $ret."\n";
+  #print $cmd1."\n";
+  if (exists($dbcfg->{$db}->{minfiles}) && $dbcfg->{$db}->{minfiles} gt "1"){
+    my $cmd2 = 'ls -1 '.$dbcfg->{$db}->{bkppath}.'/*.sql.gz  | wc -l';
+    my $num = `$cmd2`;
+    chomp($num);
+    if ($num gt $dbcfg->{$db}->{minfiles}){
+      my $cmd3 = 'find "'.$dbcfg->{$db}->{'bkppath'}.'" -name "*.sql.gz" | sort -r | sed -e \'1,'.$dbcfg->{$db}->{minfiles}.'d\' | xargs rm';
+      my $ret = `$cmd3`;
+    }
+    
+    #print $ret."\n";
+  }
+     
+}
+close(LOG);
diff --git a/bin/db_backup.sh b/bin/db_backup.sh
deleted file mode 100644
index 05f93d2..0000000
--- a/bin/db_backup.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-#CFGFILE="/mnt/c/Users/ksaff/Workspace/dksnas/.dksnas/db_backup.json"
-CFGFILE="/home/dks/backup/webserver/conf/db_backup.json"
-LOGFILE="/home/dks/backup/webserver/log/db_backup.log"
-CFGDIR=`dirname ${CFGFILE}`;
-KEYS=`cat ${CFGFILE} | jq -c keys | tr -d '[]"' | sed 's/,/\n/g'`
-IFS='
-'
-SDB=( ${KEYS} )
-for cdb in "${SDB[@]}"
-do
-  echo  `date`": backup DB : $cdb" >> ${LOGFILE}
-  DATA=`cat ${CFGFILE} | jq -c ".$cdb" | tr -d '{}"' | sed 's/,/\n/g' | sed 's/:/=/'`
-  echo ${DATA} > ${CFGDIR}'/cbackupvars.sh';
-  source ${CFGDIR}'/cbackupvars.sh';
-  rm ${CFGDIR}'/cbackupvars.sh'
-  CDATE=`date +%Y%m%d_%H%M%S`
-  OUTFILE=${bkppath}"/"$cdb"_"${CDATE}".sql"
-  #echo ${OUTFILE}
-  if [ ! -d "${bkppath}" ]
-  then
-    mkdir -p ${bkppath}
-  fi
-  if [ "${type}" = "mysql" ]
-  then
-    mysqldump -h ${dbhost} -u ${dbuser} -p${dbpwd} ${dbname} > ${OUTFILE}
-    gzip ${OUTFILE};
-  fi
-  if [ "${type}" = "pgsql" ]
-  then
-    pg_dump -F p -O -x -w ${dbname} > ${OUTFILE}
-    gzip ${OUTFILE};
-  fi
-  #find "${bkppath}" -name "*.sql.gz" | sort -r | sed -e '1,${minfiles}d' | xargs rm
-done
-exit 0
-
-
-
diff --git a/bin/file_backup.pl b/bin/file_backup.pl
new file mode 100644
index 0000000..580e372
--- /dev/null
+++ b/bin/file_backup.pl
@@ -0,0 +1,162 @@
+#!/usr/bin/perl
+use strict;
+use Net::OpenSSH;
+use File::Basename;
+use Data::Dumper;
+use Getopt::Long;
+use Archive::Tar;
+use POSIX qw/strftime/;
+use JSON::PP;
+#use Data::Dumper;
+my $strhost = ""; 
+my $full = 0;
+GetOptions("host=s" => \$strhost, "full"=> \$full);
+my $cfgpath="/home/dks/backup/webserver/conf";
+my $logfile="/home/dks/backup/webserver/log/files.".strftime('%Y%m',localtime()).'.log';
+
+opendir(CGD,$cfgpath);
+my $fcfg = ();
+while (my $f = readdir(CGD)){
+  if ($f !~ /\.files\.json$/){ next; }
+  my $file = $f;
+  $file =~ s/\.files\.json$//;
+  if ($strhost eq "" || $file =~ /^$strhost/){
+    my $strcfg = "";
+    open(CFG,$cfgpath.'/'.$f);
+    while (my $l = <CFG>){
+      $strcfg .= $l; 
+    }
+    close(CFG);
+    $fcfg->{$file} = JSON::PP::decode_json($strcfg);
+  }
+}
+closedir(CGD);
+
+open(LOG,">>".$logfile);
+foreach my $fp (keys(%{$fcfg})){
+  #if ($fp ne "saffran_lu"){ next;}
+  # print "Backup Files $fp\n";
+  print LOG "==\n".strftime("%Y-%m-%d %H:%M",localtime()).": backup Files : $fp\n";
+  my @cmd = ();
+  my $tarfile = $fp.'_'.strftime("%Y%m%d_%H%M%S",localtime()).'.tar.gz';
+  my $txtfilelist = $fp.'_'.strftime("%Y%m%d_%H%M%S",localtime()).'.txt';
+  if (! -d $fcfg->{$fp}->{'bkppath'}){
+    mkdir($fcfg->{$fp}->{'bkppath'});
+  }
+  my $fcnt = 0;
+  my $lastbackup = "";
+  opendir(DIR,$fcfg->{$fp}->{'bkppath'});
+  while (my $d = readdir(DIR)){
+    # print $d."\n";
+    if ($d =~ /\.tar\.gz$/){
+      $fcnt++;
+      my ($bkpdate) = $d =~ m/.+_(\d{8,})_\d{6,}\.tar\.gz$/;
+      # print "bdate:".$bkpdate."\n";
+      chomp($bkpdate);
+      if ($bkpdate gt $lastbackup){
+        $lastbackup = $bkpdate;
+      }  
+    }
+    
+  }
+  if ($lastbackup != ""){
+    print LOG "Last Backup made $lastbackup\n";
+  } else {
+    print LOG "No Last Backup made\n";
+  }
+  closedir(DIR);
+  if (($full == 1) || ($fcnt == 0) || ($lastbackup eq "")){
+    print LOG "Download Full\n";
+    print LOG "Create TAR\n";
+    my $stx = &execremote($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},'cd '.$fcfg->{$fp}->{remote_basepath}.' && tar czf '.$tarfile.' '.$fcfg->{$fp}->{remote_path});
+    if ($stx == 0){
+         print LOG "Download File\n";
+         my $ftx = &getfile($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},$fcfg->{$fp}->{remote_basepath}.'/'.$tarfile,$fcfg->{$fp}->{bkppath}.'/'.$tarfile);
+         if ($ftx != undef){
+           print LOG "Remove from remote\n";
+           &execremote($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},'rm '.$fcfg->{$fp}->{remote_basepath}.'/'.$tarfile);
+         }
+    }
+  } else {
+    print LOG "Download Only Changes\n";
+    my $cmd = 'cd '.$fcfg->{$fp}->{remote_basepath}.' && find '.$fcfg->{$fp}->{remote_path}.' -type f > '.$txtfilelist;
+    my $ltx = &execremote($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},$cmd);
+    $cmd = 'cd '.$fcfg->{$fp}->{remote_basepath}.' && touch -t '.$lastbackup."0000 checkdate.txt && find ".$fcfg->{$fp}->{remote_path}.' -newer checkdate.txt -type f -print | tar -czf '.$tarfile.' '.$txtfilelist.' --files-from -';
+    my $stx = &execremote($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},$cmd);
+    if ($stx == 0){
+      print LOG "Download File\n";
+      my $ftx = &getfile($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},$fcfg->{$fp}->{remote_basepath}.'/'.$tarfile,$fcfg->{$fp}->{bkppath}.'/'.$tarfile);
+      if ($ftx != undef){
+        print "Remove from remote\n";
+        &execremote($fcfg->{$fp}->{host},$fcfg->{$fp}->{ssh},'rm '.$fcfg->{$fp}->{remote_basepath}.'/'.$tarfile." && rm ".$fcfg->{$fp}->{remote_basepath}.'/'.$txtfilelist);
+      }
+      $cmd = 'tar -tzvf '.$fcfg->{$fp}->{bkppath}.'/'.$tarfile.' | wc -l';
+      my $size = `$cmd`;
+      chomp($size);
+      if ($size eq "0"){
+        print LOG "file is empty! deleting...";
+        unlink($fcfg->{$fp}->{bkppath}.'/'.$tarfile);
+      }
+    }
+  }
+}
+close(LOG);
+
+sub execremote(){
+  my $host = shift;
+  my $uopts = shift;
+  my $cmd = shift;
+  print $cmd."\n";
+  #my $stx = 0;
+  my $ssh = Net::OpenSSH->new($host,%{$uopts});
+  if ($ssh->error){
+        print "Couldn't establish SSH connection: ". $ssh->error;
+        return 1;
+  }
+  my $stx =$ssh->system($cmd);
+  #returns true (1) on success
+  if ($stx == 1){
+        $stx = 0;
+  }
+  if ($ssh->error){
+        print "remote cmd failed!". $ssh->error;
+        return 1;
+  }
+  return $stx;
+}
+
+sub piperemote(){
+  my $host = shift;
+  my $uopts = shift;
+  my $cmd = shift;
+  #print "pipe cmd:".$cmd." - to:".$host."-".Dumper($uopts)."\n";
+   my $ssh = Net::OpenSSH->new($host,%{$uopts});
+  if ($ssh->error){
+        print "Couldn't establish SSH connection: ". $ssh->error;
+        return 1;
+  }
+  print $cmd."\n";
+  my $out = $ssh->capture($cmd);
+  if ($ssh->error){
+        print "remote cmd failed!". $ssh->error;
+        return $out;
+  }
+  #print $out;
+  return $out;
+}
+
+sub getfile(){
+  my $host = shift;
+  my $uopts = shift;
+  my $remote = shift;
+  my $local =shift;
+  print "SCP Copy ".$remote.' => '.$local."\n";
+  my $ssh = Net::OpenSSH->new($host,%{$uopts});
+  if ($ssh->error){
+        print "Couldn't establish SSH connection: ". $ssh->error;
+        return undef;
+  }
+  my $ret = $ssh->scp_get({},$remote,$local) or warn "or warn scp_get from $host failed: " . $ssh->error . "\n";
+  return $ret;
+}
+
-- 
2.39.5