Dump sparse file data segment

Test file

ubuntu 16.04
ext4 file system

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
$ rm data-A
$ for i in {1..1500}; do echo $i >> data-A ; done
$ filefrag -v data-A
Filesystem type is: ef53
File size of data-A is 6394 (2 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 0.. 0: 3464588.. 3464588: 1:
1: 1.. 1: 0.. 0: 1: 3464589: last,unknown_loc,delalloc,eof
$ dd if=data-A of=data-A-1 bs=4096 count=1
$ dd if=data-A of=data-A-2 bs=4096 count=1 skip=1
$ head data-A-2
1
1042
1043
1044
1045


$ cat data-A-1 data-A-2 | md5sum
30ef7f11d29b8ad47542e97ed9d2a398 -
$ cat data-A | md5sum
30ef7f11d29b8ad47542e97ed9d2a398 -

Test sparse file

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
$ rm ./data-A;for i in {1..1500}; do echo $i >> data-A ; done
$ filefrag -v data-A
Filesystem type is: ef53
File size of data-A is 6393 (2 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 0.. 1: 0.. 1: 2: last,unknown_loc,delalloc,eof
data-A: 1 extent found

$ dd if=/dev/zero of=test bs=4096 count=0 seek=25 #write 100K sparse file
$ cat data-A >> test
$ ls -lhs
total 16K
8.0K -rw-rw-r-- 1 homerl homerl 6.3K 12月 19 14:04 data-A
8.0K -rw-rw-r-- 1 homerl homerl 107K 12月 19 14:06 test

$ df -h /
Filesystem Size Used Avail Use% Mounted on
/dev/sda2 101G 35G 61G 37% /

$ sudo dumpe2fs /dev/sda2 | grep -i "Block size"
dumpe2fs 1.42.13 (17-May-2015)
Block size: 4096


$ filefrag -v data-A
Filesystem type is: ef53
File size of data-A is 6393 (2 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 0.. 1: 3713508.. 3713509: 2: last,eof
data-A: 1 extent found

$ sudo dd if=/dev/sda2 of=test-1 bs=4096 count=1 skip=3713508
$ head test-1
1
2
3
4
5
6

$ tail test-1
1039
1040
104

$ cat test-1 | md5sum
27260c41d34d5a01f5fba073f9059a90 -

$ filefrag -v test
Filesystem type is: ef53
File size of test is 108793 (27 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 25.. 26: 3713510.. 3713511: 2: last,eof
test: 1 extent found

$ sudo dd if=/dev/sda2 of=test-s-1 bs=4096 count=1 skip=3713510

$ md5sum test-*
27260c41d34d5a01f5fba073f9059a90 test-1
27260c41d34d5a01f5fba073f9059a90 test-s-1

$ sudo dd if=/dev/sda2 of=test-s-2 bs=4096 count=1 skip=3713511

# the checksum difference between test-s-2 and test
$ hexdump test-s-2 | tail
0000890 3431 3038 310a 3834 0a31 3431 3238 310a
00008a0 3834 0a33 3431 3438 310a 3834 0a35 3431
00008b0 3638 310a 3834 0a37 3431 3838 310a 3834
00008c0 0a39 3431 3039 310a 3934 0a31 3431 3239
00008d0 310a 3934 0a33 3431 3439 310a 3934 0a35
00008e0 3431 3639 310a 3934 0a37 3431 3839 310a
00008f0 3934 0a39 3531 3030 000a 0000 0000 0000
0000900 0000 0000 0000 0000 0000 0000 0000 0000
*
0001000

$ hexdump test | tail
001a870 0a33 3431 3437 310a 3734 0a35 3431 3637
001a880 310a 3734 0a37 3431 3837 310a 3734 0a39
001a890 3431 3038 310a 3834 0a31 3431 3238 310a
001a8a0 3834 0a33 3431 3438 310a 3834 0a35 3431
001a8b0 3638 310a 3834 0a37 3431 3838 310a 3834
001a8c0 0a39 3431 3039 310a 3934 0a31 3431 3239
001a8d0 310a 3934 0a33 3431 3439 310a 3934 0a35
001a8e0 3431 3639 310a 3934 0a37 3431 3839 310a
001a8f0 3934 0a39 3531 3030 000a
001a8f9

There are some of zero fill in the file bottom.
astersik (*) !(means same as the line above)[http://superuser.com/questions/494245/what-does-an-asterisk-mean-in-hexdump-output]

dd update

operate single byte

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
$ dd if=data-A of=data-A-1Byte bs=1  count=1 ibs=1 obs=1
$ dd if=data-A of=data-A-skip-1byte bs=1 skip=2 count=1 ibs=1 obs=1 iflag=count_bytes iflag=skip_bytes

$ ls -ls | grep -i by

#first column is KiB , 4 is 4KiB
#sixth column is byte , 1 byte

4 -rw-rw-r-- 1 homerl homerl 1 12月 19 15:23 data-A-1Byte
4 -rw-rw-r-- 1 homerl homerl 1 12月 19 15:22 data-A-skip-1byte

$ cat data-A-1Byte
1 $ cat data-A-skip-1byte
2

$ ls -ls test data-A
8 -rw-rw-r-- 1 homerl homerl 6393 12月 19 14:54 data-A
8 -rw-rw-r-- 1 homerl homerl 108793 12月 19 15:30 test

$ dd if=test of=test-data bs=1 count=6393 skip=102400 iflag=skip_bytes ibs=1 obs=1
$ $ sha1sum data-A test-data test
8f8374832cb6536f580eef3b316ad08293a47617 data-A
8f8374832cb6536f580eef3b316ad08293a47617 test-data
03e31af4c89c9003ee0f43e49a65ad2e24367024 test

$ hexdump data-A | tail
0001870 0a33 3431 3437 310a 3734 0a35 3431 3637
0001880 310a 3734 0a37 3431 3837 310a 3734 0a39
0001890 3431 3038 310a 3834 0a31 3431 3238 310a
00018a0 3834 0a33 3431 3438 310a 3834 0a35 3431
00018b0 3638 310a 3834 0a37 3431 3838 310a 3834
00018c0 0a39 3431 3039 310a 3934 0a31 3431 3239
00018d0 310a 3934 0a33 3431 3439 310a 3934 0a35
00018e0 3431 3639 310a 3934 0a37 3431 3839 310a
00018f0 3934 0a39 3531 3030 000a
00018f9
$ hexdump test-data | tail
0001870 0a33 3431 3437 310a 3734 0a35 3431 3637
0001880 310a 3734 0a37 3431 3837 310a 3734 0a39
0001890 3431 3038 310a 3834 0a31 3431 3238 310a
00018a0 3834 0a33 3431 3438 310a 3834 0a35 3431
00018b0 3638 310a 3834 0a37 3431 3838 310a 3834
00018c0 0a39 3431 3039 310a 3934 0a31 3431 3239
00018d0 310a 3934 0a33 3431 3439 310a 3934 0a35
00018e0 3431 3639 310a 3934 0a37 3431 3839 310a
00018f0 3934 0a39 3531 3030 000a
00018f9
$ hexdump test | tail
001a870 0a33 3431 3437 310a 3734 0a35 3431 3637
001a880 310a 3734 0a37 3431 3837 310a 3734 0a39
001a890 3431 3038 310a 3834 0a31 3431 3238 310a
001a8a0 3834 0a33 3431 3438 310a 3834 0a35 3431
001a8b0 3638 310a 3834 0a37 3431 3838 310a 3834
001a8c0 0a39 3431 3039 310a 3934 0a31 3431 3239
001a8d0 310a 3934 0a33 3431 3439 310a 3934 0a35
001a8e0 3431 3639 310a 3934 0a37 3431 3839 310a
001a8f0 3934 0a39 3531 3030 000a
001a8f9

Expand sparse file

1
2
3
4
5
6
7
8
9
10
$ truncate -s +256 test_bak
$ ls -ls test_bak
108 -rw-rw-r-- 1 homerl homerl 109049 12月 19 16:36 test_bak
$ ls -ls test
8 -rw-rw-r-- 1 homerl homerl 108793 12月 19 15:30 test
$ echo $((109049-108793))
256

# not sparse
$fallocate -o 1M -l 1M 1M-file

About fallocate

collapse-range

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
$ rm test;for i in {0..60000}; do echo $i >> test; done
$ ls -ls test
344 -rw-rw-r-- 1 homerl homerl 348896 12月 19 17:03 test
$ cp test test2
$ fallocate --collapse-range --offset 4096 --length 8192 test2 # From 4096, delete length from 4096 delete 8192 length bytes
$ dd if=test of=test-dd-4096-8192 bs=4096 count=2 skip=4096 iflag=skip_bytes # make sure delete length

head test-dd-4096-8192
041
1042
1043
1044
1045
1046
1047
1048
1049
1050

$ tail test-dd-4096-8192
2670
2671
2672
2673
2674
2675
2676
2677
2678
267

$ head -n 1045 test2 | tail
1035
1036
1037
1038
1039
1040
19 # 1 041 and 267 9
2680
2681
2682

# test in sparse file
$ dd if=/dev/zero of=test-sparse bs=1M count=0 seek=1
$ ls -lsh
total 344K
344K -rw-rw-r-- 1 homerl homerl 341K 12月 19 17:03 test
0 -rw-rw-r-- 1 homerl homerl 1.0M 12月 19 17:23 test-sparse
$ cat test >> test-sparse
$ ls -ls
total 688
344 -rw-rw-r-- 1 homerl homerl 348896 12月 19 17:03 test
344 -rw-rw-r-- 1 homerl homerl 1397472 12月 19 17:23 test-sparse

$ fallocate --collapse-range --offset 4096 --length 8192 test-sparse
$ ls -ls
total 688
344 -rw-rw-r-- 1 homerl homerl 348896 12月 19 17:03 test
344 -rw-rw-r-- 1 homerl homerl 1389280 12月 19 17:24 test-sparse

$ filefrag -v test-sparse
Filesystem type is: ef53
File size of test-sparse is 1397472 (342 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 254.. 341: 3803866.. 3803953: 88: last,eof

$ filefrag -v test-sparse
Filesystem type is: ef53
File size of test-sparse is 1389280 (340 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 252.. 339: 3803866.. 3803953: 88: last,eof
test-sparse: 1 extent found

zero-range

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
$ cp  test test-sparse 
$ dd if=/dev/zero of=test-sparse bs=1M count=0 seek=1
0+0 records in
0+0 records out
0 bytes copied, 0.000132755 s, 0.0 kB/s
1d [homerl:/tmp/test/test2] 130 $ filefrag -v test-sparse
Filesystem type is: ef53
File size of test-sparse is 1048576 (256 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 0.. 2: 589896.. 589898: 3: last,unwritten
test-sparse: 1 extent found
1d [homerl:/tmp/test/test2] $ cat test >> test-sparse
1d [homerl:/tmp/test/test2] $ filefrag -v test-sparse
Filesystem type is: ef53
File size of test-sparse is 1397472 (342 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 0.. 2: 589896.. 589898: 3: unwritten
1: 256.. 341: 0.. 85: 86: 589899: last,unknown_loc,delalloc,eof
test-sparse: 2 extents found
$ head test-sparse
0
1
2
3
4
5
6
7
8
9

$ fallocate --zero-range --offset 1048576 --length 128 test-sparse
$ head test-sparse
46
47
48
49
50
51
52
53
54
55

$ fallocate --zero-range --offset 1048576 --length 348896 test-sparse
# clear all

punch-hole

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
$ cat test >> test-sparse 
$ filefrag -v test-sparse
Filesystem type is: ef53
File size of test-sparse is 1397472 (342 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 256.. 341: 0.. 85: 86: last,unknown_loc,delalloc,eof
test-sparse: 1 extent found


$cat test >> test-sparse
1d [homerl:/tmp/test/test2] $ ls -ls
total 688
344 -rw-rw-r-- 1 homerl homerl 348896 12月 19 17:29 test
344 -rw-rw-r-- 1 homerl homerl 1397472 12月 19 17:53 test-sparse

$ echo ((4096*256)=1048576 ;((4096*84))=344064

$fallocate --punch-hole --offset 1048576 --length 344064 test-sparse
1d [homerl:/tmp/test/test2] $ filefrag -v test-sparse
Filesystem type is: ef53
File size of test-sparse is 1397472 (342 blocks of 4096 bytes)
ext: logical_offset: physical_offset: length: expected: flags:
0: 340.. 341: 3802396.. 3802397: 2: last,eof
test-sparse: 1 extent found

1d [homerl:/tmp/test/test2] $ ls -ls test-sparse
8 -rw-rw-r-- 1 homerl homerl 1397472 12月 19 17:54 test-sparse

# space has recycled