13.4 Tâches CRON et autres

Récupération des données depuis MongoDB ATLAS et backup des données (note : remplacer d’abord les items sensibles comme ou par le mot de passe et le nom du serveur MongoDB Atlas !!!) :

echo $'#!/bin/bash
# Recuperation of MongoDB ATLAS data into local MongoDB database
# must be executed as root
# Copyright (c) 2021, Philippe Grosjean (phgrosjean@sciviews.org)

# First check that major versions of MongoDB servers (ATLAS vs local, x.y)
# match (otherwise, we may get corrupted data!)
mongo_local_version=`mongo --quiet --eval "db.version()" | awk -F "." \'{print $1 "." $2}\'`
mongo_atlas_version=`mongo --quiet --eval "db.version()" "mongodb://<USER>:<PASSWORD>@<SERVER>.mongodb.net:27017,<SERVER2>.mongodb.net:27017,<SERVER3>.mongodb.net:27017/sdd?ssl=true&replicaSet=<SHARD>&authSource=admin" | awk -F "." \'{print $1 "." $2}\'`
if [ "$mongo_local_version" != "$mongo_atlas_version" ]; then
  echo "MongoDB versions mismatch: local=$mongo_local_version, ATLAS=$mongo_atlas_version" > /data1/VERSIONS_MISMATCH
  exit 0 # Use a higher value to report the error
else
  rm -f /data1/VERSIONS_MISMATCH
fi

# Make sure /data1/dump directory exists
cd /data1 &&
mkdir -p /data1/dump &&
chown <USER>:<GROUP> /data1/dump &&
# Record current date (in UTC format)
date -u "+%Y-%m-%d %H:%M:%S" > cur_date &&
# Is it a last_date file?
if [ -f "last_date" ]; then
  # Last date file exists, use it and get only data from that date
  # Need to specify collection here => call 3x for h5p, shiny & learnr
  mongodump --quiet --uri="mongodb://<USER>:<PASSWORD>@<SERVER>.mongodb.net:27017,<SERVER2>-umnnw.mongodb.net:27017,<SERVER3>.mongodb.net:27017/sdd?ssl=true&replicaSet=<SHARD>&authSource=admin" --db="sdd" --collection="h5p" --query="{\\\"date\\\":{\\\"\$gte\\\":\\\"$(cat last_date)\\\"}}"
  mongodump --quiet --uri="mongodb://<USER>:<PASSWORD>@<SERVER>.mongodb.net:27017,<SERVER1>.mongodb.net:27017,<SERVER2>.mongodb.net:27017/sdd?ssl=true&replicaSet=<SHARD>&authSource=admin" --db="sdd" --collection="shiny" --query="{\\\"date\\\":{\\\"\$gte\\\":\\\"$(cat last_date)\\\"}}"
  mongodump --quiet --uri="mongodb://<USER>:<PASSWORD>@<SERVER>.mongodb.net:27017,<SERVER2>.mongodb.net:27017,<SERVER3>.mongodb.net:27017/sdd?ssl=true&replicaSet=<SHARD>&authSource=admin" --db="sdd" --collection="learnr" --query="{\\\"date\\\":{\\\"\$gte\\\":\\\"$(cat last_date)\\\"}}"
else
  # No last_date file, get everything from MongoDB ATLAS
  mongodump --quiet --uri="mongodb://<USER>:<PASSWORD>@<SERVER>.mongodb.net:<SERVER2>-umnnw.mongodb.net:27017,<SERVER3>.mongodb.net:27017/sdd?ssl=true&replicaSet=<SHARD>&authSource=admin" --db="sdd"
fi

# If something is collected, an sdd subdirectory is created
if [ -d "/data1/dump/sdd" ]; then
  # Backup last_restore
  if [ -f "last_restore" ]; then
    cp last_restore forelast_restore
  fi
  # Backup last_date too
  if [ -f "last_date" ]; then
    cp last_date forelast_date
  fi
  mongorestore &> last_restore &&
  cp cur_date last_date
fi

# Clean up /data1/dump directory and go back to intial dir
rm -rf /data1/dump/* &&
cd - > /dev/null
' | sudo tee /usr/local/bin/get_atlas_data > /dev/null &&
sudo chmod 755 /usr/local/bin/get_atlas_data &&
sudo chmod +x /usr/local/bin/get_atlas_data

Réaliser un backup quotidien roulant, ainsi qu’un backup permanent chaque samedi :

# Regular daily backup 
echo '#!/bin/bash
# Daily, temporary MongoDB database backup (overwritten next same day)
DIR=`date +%a` &&
DEST=/data1/backup/mongodb/$DIR &&
rm -rf $DEST &&
mkdir -p $DEST &&
mongodump -o $DEST &&
date -u "+%Y-%m-%d %H:%M:%S" > $DEST/backup_date
' | sudo tee /usr/local/bin/backup_data_daily > /dev/null &&
sudo chmod 755 /usr/local/bin/backup_data_daily &&
sudo chmod +x /usr/local/bin/backup_data_daily &&

# Weekly, more permanent backup
echo '#!/bin/bash
# Weekly, permanent MongoDB database backup
DIR=`date +%Y%m%d` &&
DEST=/data1/backup/mongodb/$DIR &&
mkdir -p $DEST &&
mongodump -o $DEST
' | sudo tee /usr/local/bin/backup_data_weekly > /dev/null &&
sudo chmod 755 /usr/local/bin/backup_data_weekly &&
sudo chmod +x /usr/local/bin/backup_data_weekly

Vérifier ces scripts et ensuite, ajouter une entrée crontab :

sudo crontab -e

Éditer le fichier en y rajoutant :

# Get data from MongoDB ATLAS every 5 min
*/5 * * * * /usr/local/bin/get_atlas_data
# Sync /data1/ with /data2/ every hour:47
47 * * * * /usr/bin/rsync -av --delete /data1/ /data2/ &> /data/datasync.log
# Create a daily (except saterday) and a weekly backup of the MongoDB database
# on 03:41:00
41 3 * * 6 /usr/local/bin/backup_data_weekly
41 3 * * 0,1,2,3,4,5 /usr/local/bin/backup_data_daily