Eigenen Cloudspeicher mit dem Raspberry

Anforderungen

Warum keine Fertiglösung?

Lessons learned:

• Niemals ein Array auf nicht eindeutigen Identifiern anlegen wie z.B. /dev/sda. Bei meinem USB Laufwerken hatte dies fatale folgen.

Raid 5 mit 3 x HDDs + 1 x SSD

CACHE_DISK=/dev/disk/by-id/usb-SanDisk_SSD_PLUS_480GB_152D00539000-0:3

DATA_DISK_1=/dev/disk/by-id/usb-SAMSUNG_HD154UI_152D00539000-0:0
DATA_DISK_2=/dev/disk/by-id/usb-SAMSUNG_HD154UI_152D00539000-0:1
DATA_DISK_3=/dev/disk/by-id/usb-SAMSUNG_HD154UI_152D00539000-0:2

DATA_RAID_LABEL=raid-data
DATA_RAID_NODE=/dev/md/${DATA_RAID_LABEL}

VOLUME_GROUP=vg1
LOGICAL_VOLUME_PREFIX=lv_data1

CACHE_MODE=writeback

hdparm -W1 ${CACHE_DISK}

hdparm -W0 ${DATA_DISK_1}
hdparm -W0 ${DATA_DISK_2}
hdparm -W0 ${DATA_DISK_3}

mdadm --create ${DATA_RAID_LABEL} --level=5 --raid-devices=3 ${DATA_DISK_1} ${DATA_DISK_2} ${DATA_DISK_3}
mdadm --detail --scan ${DATA_RAID_NODE} >> /etc/mdadm.conf

pvcreate ${DATA_RAID_NODE}
pvcreate ${CACHE_DISK} # is this required?

vgcreate ${VOLUME_GROUP} "${DATA_RAID_NODE}" "${CACHE_DISK}"

lvcreate  --size 1500G  --name ${LOGICAL_VOLUME_PREFIX}             ${VOLUME_GROUP}   "${DATA_RAID_NODE}"
lvcreate  -L 450G   --name ${LOGICAL_VOLUME_PREFIX}_cache       ${VOLUME_GROUP}   "${CACHE_DISK}"
lvcreate  -L 1.6G   --name ${LOGICAL_VOLUME_PREFIX}_cache_meta  ${VOLUME_GROUP}   "${CACHE_DISK}"

lvconvert \
  --type cache-pool \
  --cachemode ${CACHE_MODE} \
  --poolmetadata ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache_meta \
  ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache

lvconvert \
  --type cache \
  --cachepool ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache \
  ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}

Testing mit 1 x HDD + 1 x SSD

CACHE_DISK=/dev/sde
DATA_DISK_1=/dev/sdd

VOLUME_GROUP=vg1
LOGICAL_VOLUME_PREFIX=lv_data1

CACHE_MODE=writeback

hdparm -W1 ${CACHE_DISK}
hdparm -W0 ${DATA_DISK_1}

pvcreate ${DATA_DISK_1}
pvcreate ${CACHE_DISK} # is this required?

vgcreate ${VOLUME_GROUP} "${DATA_DISK_1}" "${CACHE_DISK}"

lvcreate  -L 1400G  -n ${LOGICAL_VOLUME_PREFIX}             ${VOLUME_GROUP}   "${DATA_DISK_1}"
lvcreate  -L 450G   -n ${LOGICAL_VOLUME_PREFIX}_cache       ${VOLUME_GROUP}   "${CACHE_DISK}"
lvcreate  -L 1.6G   -n ${LOGICAL_VOLUME_PREFIX}_cache_meta  ${VOLUME_GROUP}   "${CACHE_DISK}"

lvconvert \
  --type cache-pool \
  --cachemode ${CACHE_MODE} \
  --poolmetadata ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache_meta \
  ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache

lvconvert \
  --type cache \
  --cachepool ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache \
  ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}

Partitioning and mount

fdisk /dev/mapper/${VOLUME_GROUP}-${LOGICAL_VOLUME_PREFIX} <<EOF
g
n
1

w
EOF
partprobe /dev/mapper/${VOLUME_GROUP}-${LOGICAL_VOLUME_PREFIX}
mkfs.ext4 /dev/mapper/${VOLUME_GROUP}-${LOGICAL_VOLUME_PREFIX}p1
mount /dev/mapper/${VOLUME_GROUP}-${LOGICAL_VOLUME_PREFIX}p1 /mnt/data

Testen von Performance und Caching

dd

Undo steps

fdisk /dev/mapper/${VOLUME_GROUP}-${LOGICAL_VOLUME_PREFIX} <<EOF
g
w
EOF
partprobe /dev/mapper/${VOLUME_GROUP}-${LOGICAL_VOLUME_PREFIX}

lvconvert --splitcache ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}
lvremove ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}
lvremove ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache
lvremove ${VOLUME_GROUP}/${LOGICAL_VOLUME_PREFIX}_cache_meta
vgremove ${VOLUME_GROUP}
pvremove ${DATA_RAID_NODE}
pvremove ${CACHE_DISK}

Raspberry

# Create large/slow data volume on mechanical disk
pvcreate /dev/sdb
vgcreate vg-data /dev/sdb
lvcreate --name data -l 100%FREE vg-data /dev/sdb
mkfs.ext4 /dev/vg-data/data
mount /dev/vg-data/data /media/data/

# Create smaller/fast volume to store random access data (e.g. mysql DB and docker volumes)
pvcreate /dev/sda
vgextend vg-data /dev/sda
lvcreate -L 100G -n docker vg-data /dev/sda
mkfs.ext4 /dev/vg-data/docker

# Use the remaining free disk space on fast volume to create disk cache for large volume
lvcreate --name data-cache --type cache-pool -l 100%FREE vg-data /dev/sda
lvconvert --type cache --cache-pool data-cache --cachemode writeback vg-data/data

Aktivieren / Deaktivieren

lvconvert --splitcache vg-data/data
lvconvert --type cache --cachemode writethrough --cachepool cache vg-data/data