Install robinhood

Just analyse lustre data, not remove or move any file and directory.
And I created zfs pool in /var/lib/mysql, and enable lz4 compression.
Now here is compressratio

1
2
3
# zfs get compressratio tank
NAME PROPERTY VALUE SOURCE
tank compressratio 2.50x -

CentOS 7.2 and Robinhood 2.5.5

1
2
3
4
5
6
7
8
# yum -y install mariadb-devel libuuid-devel mailx glib2-devel libattr-devel mariadb-server attr
# tar xzvf robinhood-2.5.5.tar.gz
# cd robinhood
# ./configure --with-purpose=TMPFS
or
# ./configure --with-purpose=LUSTRE_HSM
# make rpm
# rpm -ivh robinhood-tmpfs-2.5.5-4.lustre2.5.el7.centos.x86_64 robinhood-adm-2.5.5-4.noarch.x86_64

Init mariadb

1
2
3
4
# systemctl enable mariadb
# systemctl start mariadb
# mysqladmin -u root password "your root password";
# rbh-config create_db

Modify /etc/robinhood.d/tmpfs/templates/tmpfs_detailed.conf as your wish, and save it to /etc/robinhood.d/tmpfs/tmpfs.conf.

Enable lustre changelog

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
ChangeLog
{
# 1 MDT block for each MDT :
MDT
{
mdt_name = "MDT0000" ;
reader_id = "cl1" ;
}

# clear changelog every 1024 records:
batch_ack_count = 1024 ;

force_polling = ON ;
polling_interval = 1s ;
queue_max_size = 1000 ;
queue_max_age = 5s ;
queue_check_interval = 1s ;
dump_file = "/var/lib/mysql/robinhood/changelog_dump.log";
}

# lctl --device lustrefs-MDT0000 changelog_register
lustrefs-MDT0000: Registered changlog userid 'cl1'
# lctl set_param mdd.lustrefs-MDT*.changelog_mask "MARK CREAT MKDIR HLINK SLINK MKNOD UNLNK RMDIR RENME RNMTO OPEN LYOUT TRUNC SATTR XATTR HSM MTIME CTIME"
# lctl get_param mdd.lustrefs-MDT*.changelog_mask
MARK CREAT MKDIR HLINK SLINK MKNOD UNLNK RMDIR RENME RNMTO OPEN LYOUT TRUNC SATTR XATTR HSM MTIME CTIME

Run robinhood

write IOPS peak about 9000~10000 in single 10GbE SR SFP+ port.

1
robinhood -f /etc/robinhood.d/tmpfs/tmpfs.conf --read-log -C --dry-run --scan

Show data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# rbh-report --top-users -U 10000
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
rank, user , spc_used, count, avg_size
1, 501 , 142.80 TB, 593083, 252.58 MB
2, 35061 , 33.82 TB, 50912, 696.54 MB
3, 35623 , 29.91 TB, 42304, 741.30 MB
4, 666666 , 26.17 TB, 300038, 91.44 MB
5, 35939 , 16.26 TB, 1616106, 10.54 MB
6, 31344 , 15.63 TB, 801339, 20.45 MB
7, 30428 , 13.54 TB, 272051, 52.19 MB

# rbh-report --top-users --by-count
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
rank, user , spc_used, count, avg_size
1, 28904 , 778.50 GB, 4493723, 179.10 KB
2, 504 , 4.14 TB, 2514345, 1.73 MB
3, 29637 , 2.81 TB, 2152429, 1.37 MB
4, 29563 , 873.35 GB, 1935333, 470.53 KB
5, 30185 , 508.22 GB, 1695768, 310.95 KB
6, 35939 , 16.26 TB, 1616106, 10.54 MB

# rbh-du -sH /lustrefs/share_backup/ -u 30319 --details
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
ls
total
symlink count:895, size:46.3K, spc_used:1.4M
dir count:9877, size:42.7M, spc_used:42.7M
file count:128990, size:5.2T, spc_used:5.2T

# rbh-du -H -u 30319
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
5.2T /lustrefs

# rbh-report -u 30319 -S
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
user , group, type, count, spc_used, avg_size
30319 , 725, symlink, 5, 4.00 KB, 36
30319 , 725, dir, 811, 3.22 MB, 4.07 KB
30319 , 725, file, 4388, 5.00 TB, 1.17 GB
30319 , 820, symlink, 13, 0, 35
30319 , 820, dir, 23, 92.00 KB, 4.00 KB
30319 , 820, file, 287, 7.78 GB, 27.77 MB
30319 , 859, symlink, 881, 1.38 MB, 53
30319 , 859, dir, 9066, 39.48 MB, 4.45 KB
30319 , 859, file, 124457, 208.88 GB, 1.72 MB
30319 , root, dir, 1, 4.00 KB, 4.00 KB

# rbh-report -u 30319 --szprof
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
user , type, count, spc_used, avg_size, 0, 1~31, 32~1K-, 1K~31K, 32K~1M-, 1M~31M, 32M~1G-, 1G~31G, 32G~1T-, +1T
30319 , symlink, 899, 1.39 MB, 53, 0, 357, 542, 0, 0, 0, 0, 0, 0, 0
30319 , dir, 9902, 42.80 MB, 4.42 KB, 0, 0, 0, 9875, 27, 0, 0, 0, 0, 0
30319 , file, 129132, 5.21 TB, 42.31 MB, 2288, 2300, 31102, 75173, 14854, 2366, 863, 100, 86, 0

Total: 139933 entries, 5728992747520 bytes used (5.21 TB)

# rbh-report --top-users --by-szratio=1G..31G
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
rank, user , spc_used, count, avg_size, 0, 1~31, 32~1K-, 1K~31K, 32K~1M-, 1M~31M, 32M~1G-, 1G~31G, 32G~1T-, +1T, ratio(1G..31G)
1, 29862 , 8.96 TB, 10238, 918.14 MB, 9, 2, 1729, 2795, 466, 355, 679, 4185, 18, 0, 40.88%
2, 35590 , 37.44 GB, 39, 983.16 MB, 2, 8, 10, 5, 0, 1, 0, 13, 0, 0, 33.33%
3, 35591 , 17.47 GB, 22, 812.97 MB, 0, 0, 10, 3, 0, 1, 2, 6, 0, 0, 27.27%
4, qemu , 647.41 GB, 127, 5.27 GB, 0, 0, 5, 39, 3, 39, 6, 31, 4, 0, 24.41%
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# rbh-report --top-dirs --by-count
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.
rank, path, dircount, avgsize, user, group, last_mod
1, /lustrefs/cm/net, 1365749, 56.84 KB, 35697, 904, 2016/02/01 03:58:45
2, /lustrefs/tte/net, 443993, 54.27 KB, 35697, 904, 2016/01/25 12:24:21
3, /lustrefs/tte/crf/crf-net, 443993, 39.71 KB, 35697, 904, 2016/01/26 03:04:40
4, /lustrefs/LabelFree/net, 387909, 82.25 KB, 35697, 904, 2016/01/26 02:36:44
5, /lustrefs/LabelFree/crf/crf-net, 387909, 50.98 KB, 35697, 904, 2016/01/26 06:02:10

# rbh-find /lustrefs/test/ -user 501 -size +1G –ost 4

# rbh-find -user 501 -print
/lustrefs/xxxx
/lustrefs/xxx/xxx
......
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# rbh-report -a
Using config file '/etc/robinhood.d/tmpfs/tmpfs.conf'.

Filesystem scan activity:

Current scan interval: 7.0d

Previous filesystem scan:
start: 2016/02/17 21:30:40
duration: 02s

Last filesystem scan:
status: done
start: 2016/02/17 21:30:43
end: 2016/02/18 09:32:36
duration: 12h 01min 53s

Statistics:
entries scanned: 58038696
errors: 1
timeouts: 0
# threads: 24
average speed: 1450.15 entries/sec


Changelog stats:

Last read record id: 5151588
Last read record time: 2016/02/19 01:29:42.783530
Last receive time: 2016/02/19 01:28:56
Last committed record id: 5151588
Changelog stats:
type total (diff) (rate)
MARK: 0
CREAT: 2363800 (+30) (0.03/sec)
MKDIR: 48696
HLINK: 110
SLINK: 1740 (+2) (0.00/sec)
MKNOD: 0
UNLNK: 137955 (+161) (0.13/sec)
RMDIR: 5993 (+1) (0.00/sec)
RENME: 3798 (+5) (0.00/sec)
RNMTO: 0
OPEN: 0
CLOSE: 0
LYOUT: 0
TRUNC: 236843
SATTR: 2164457 (+19) (0.02/sec)
XATTR: 16732
HSM: 0
MTIME: 284
CTIME: 0
ATIME: 0

Storage usage has never been checked

No purge was performed on this filesystem

Install web interface

1
2
3
4
5
6
7
# rpm -ivh robinhood-webgui-2.5.5-4.noarch.x86_64.rpm
# yum -y install php php-mysql php-xml php-pdo php-gd httpd
# chown -R apache.apache /var/www/html/robinhood

## Modify httpd.conf
<Directory "/var/www/html">
AllowOverride All

Start httpd

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
systemctl enable httpd
systemctl start httpd
```

```mysql
[mysqld]
user = mysql
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
port = 3306
basedir = /usr
datadir = /var/lib/mysql
tmpdir = /tmp

# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
skip-external-locking
performance_schema
skip-innodb_doublewrite
skip-log-bin

innodb_page_size=32k
innodb_data_file_path=ibdata1:100M:autoextend
connect_timeout=60
thread_cache_size=512
innodb_file_per_table = true
log_slow_verbosity = query_plan

key_buffer_size=4G
query_cache_size=4G
query_cache_limit=4G
sort_buffer_size=8G
read_rnd_buffer_size=4G
tmp_table_size=4G
max_heap_table_size=4G

#slow log
slow_query_log = 1
slow_query_log_file = /var/lib/mysql/slow.log
long_query_time = 60
log_queries_not_using_indexes=1

# innodb_buffer_pool_size: recommended value is 80% of physical memory
innodb_buffer_pool_size = 42G
innodb_lock_wait_timeout = 300
open-files-limit=8192
max_connections=1024
innodb_io_capacity=100000
innodb_autoinc_lock_mode = 2
innodb_thread_concurrency = 0
innodb_log_buffer_size = 512M
innodb_log_file_size = 256M
innodb_log_files_in_group = 4
innodb_lock_wait_timeout = 90

innodb_max_dirty_pages_pct = 20
innodb_file_per_table = 1
innodb_write_io_threads = 8
innodb_read_io_threads = 8

default_storage_engine = InnoDB
innodb_log_file_size = 50M
innodb_log_buffer_size = 16M
innodb_file_per_table = true
innodb_open_files = 4000
innodb_lru_scan_depth = 4000
innodb_use_trim=ON
innodb_use_fallocate=ON
innodb_page_size=16k
innodb_write_io_threads=24
innodb_read_io_threads=24
innodb_buffer_pool_instances=16
innodb_mtflush_threads=16
innodb_doublewrite=0

mysql tips

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
#create user
CREATE USER 'database_user'@'localhost' IDENTIFIED BY 'user_password';

#change password
ALTER USER 'database_user'@'localhost' IDENTIFIED BY 'new_password';
SET PASSWORD FOR 'database_user'@'localhost' = PASSWORD('new_password');

#show all user
SELECT user, host FROM mysql.user;

#Grant permission to db
GRANT ALL PRIVILEGES ON database_name.* TO 'database_user'@'localhost';
GRANT ALL PRIVILEGES ON *.* TO 'database_user'@'localhost';
GRANT SELECT, INSERT, DELETE ON database_name.* TO database_user@'localhost';

#Grant permission and create user
GRANT SELECT ON robinhood_lustre.* TO user@'%' IDENTIFIED BY 'yourpassword';

#Grant permission to multiple users
GRANT SELECT,INSERT,UPDATE ON db1.* TO 'user1', 'user2', 'user3', 'user4', 'user5', 'user6';

#Grant permission to table
GRANT ALL PRIVILEGES ON database_name.table_name TO 'database_user'@'localhost';

#Revoke permission
REVOKE ALL PRIVILEGES ON database_name.* TO 'database_user'@'localhost'

#show all privileges
SHOW GRANTS FOR 'database_user'@'localhost';

#re-init mysql data, because you modify my.cnf ,eg: innodb_page_size=64k
mysql_install_db (mysql is mysqld --initialize)
chown -R mysql.mysql /var/lib/mysql

#select
select YEAR(FROM_UNIXTIME(creation_time)) FROM ANNEX_INFO where YEAR(FROM_UNIXTIME(creation_time))="2015" limit 10;
select YEAR(FROM_UNIXTIME(creation_time)) FROM ANNEX_INFO where YEAR(FROM_UNIXTIME(creation_time)) < 2016 limit 10;

#test mysql error
sudo -u mysql /usr/libexec/mysqld --skip-grant-tables --general-log

#update
update ENTRIES SET owner='1000', gr_name='1000' where owner='yourname'

#Get record counts for all tables in mysql
SELECT SUM(TABLE_ROWS) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{your_db}';

#bash in mysql
mysql db_name < text_file
mysql < thefile
mysql -u <user> -p -e "select * from schema.table"
mysql -uroot -p -hslavedb.mydomain.com mydb_production <<< 'select * from users'

Enable changelog in robinhood 3.0

1
2
3
4
$ lctl set_param mdt.$FSNAME-MDT0000.hsm_control=enabled
mdt.lustre-MDT0000.hsm_control=enabled
$ lctl set_param mdt.$FSNAME-MDT0000.hsm.max_requests=64
mdt.lustre-MDT0000.hsm.max_requests=64

Start lhsmtool_cmd

1
2
3
4
$ cat /etc/lhsm_cmd.conf
[commands]
archive = /sbin/aliclient -action=push -fd={fd} -fid={fid} -lustre_root=/opt/lustresz -objdomain=10.50.192.73 -objkeyid=iiiiiiiiiiiiidddddddddd -objkey=kkkkkkkkeeeeeeeeeeyyyyyyyyyy -objbucket=bucket-name
restore = /sbin/aliclient -action=pull -fd={fd} -fid={fid} -lustre_root=/opt/lustresz -objdomain=10.50.192.73 -objkeyid=iiiiiiiiiiiiidddddddddd -objkey=kkkkkkkkeeeeeeeeeeyyyyyyyyyy -objbucket=bucket-name

Start process

1
$ lhsmtool_cmd --abort-on-error --fanout 16 -vv /lustre/ |& tee -a /var/log/lhsmtool_cmd.log

Robinhood configuration

Please reference source code lhsm.conf

Manually test

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
$ du -sh ./test1.fasta
68M ./test1.fasta
$ lfs hsm_state test1.fasta
test1.fasta: (0x00000000)
$ lfs hsm_archive test1.fasta
$ lfs hsm_state test1.fasta
test1.fasta: (0x00000009) exists archived, archive_id:1
$ du -hs test1.fasta
68M test1.fasta
$ lfs hsm_release test1.fasta
$ du -hs test1.fasta
512 test1.fasta
$ head test1.fasta
>m140928_184123_42139_c100719602550000001823155305141590_s1_p0/54498/0_10184
$ du -hs test1.fasta
68M test1.fasta

lhsmtool_cmd log file

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
$ tail -f /var/log/lhsmtool_cmd.log
1482217383.283695 lhsmtool_cmd[6020]: waiting for message from kernel
1482227311.130852 lhsmtool_cmd[6020]: copytool fs=lustresz archive#=1 item_count=1
1482227311.130919 lhsmtool_cmd[6020]: waiting for message from kernel
1482227311.133001 lhsmtool_cmd[6025]: Running ARCHIVE command: '/sbin/aliclient -action=push -fd=72 -fid=[0x200017990:0x29:0x0] -lustre_root=/opt/lustresz -objdomain=10.50.192.73 -objkeyid=zNWouvp5yGJWV9BB -objkey=9AKDku9VYhlovI1MO6OVCZWcIzMdnp -objbucket=bucket-dxjfbq-9'
--syscmd output: /opt/lustresz/alioss/test1.fasta

--begin upload
--The file size 70848886 Bytes
--uploadfile fid fd filepath: [0x200017990:0x29:0x0] 72 /opt/lustresz/alioss/test1.fasta
1482227323.543305 lhsmtool_cmd[6025]: Action completed, notifying coordinator cookie=0x5858d7a9, FID=[0x200017990:0x29:0x0], hp_flags=0 err=0
1482227323.548908 lhsmtool_cmd[6025]: llapi_hsm_action_end on '/opt/lustresz/.lustre/fid/0x200017990:0x29:0x0' ok (rc=0)
1482227667.051083 lhsmtool_cmd[6061]: Running RESTORE command: '/sbin/aliclient -action=pull -fd=74 -fid=[0x200017990:0x29:0x0] -lustre_root=/opt/lustresz -objdomain=10.50.192.73 -objkeyid=zNWouvp5yGJWV9BB -objkey=9AKDku9VYhlovI1MO6OVCZWcIzMdnp -objbucket=bucket-dxjfbq-9'
--syscmd output: /opt/lustresz/alioss/test1.fasta
--begin download
1482227668.728724 lhsmtool_cmd[6061]: Action completed, notifying coordinator cookie=0x5858d7aa, FID=[0x200017990:0x29:0x0], hp_flags=0 err=0
1482227668.776502 lhsmtool_cmd[6061]: llapi_hsm_action_end on '/opt/lustresz/.lustre/fid/0x200017990:0x29:0x0' ok (rc=0)

reference:
https://github.com/cea-hpc/robinhood/wiki/Documentation
http://cdn.opensfs.org/wp-content/uploads/2013/04/lug13-robinhood.pdf