tmux教程 功能 分屏:可以在一個開發框里分屏 允許terminal在連接斷開之後可以繼續運行,讓進程不會因為斷開連接而中斷 結構 // 一個tmux可以包含多個session,一個session可以包含多個window,一個window可以包含多個pane。 tmux: session 0: w ...
[20231204]快速刪除大量文件測試(rsync版本)4.txt
--//前個星期做了大量文件測試,發現只要順著inode節點順序刪除文件效率最高.
--//還有一種方式使用rsync,利用建立空目錄使用同步功能,刪除對應目錄裡面的全部文件.
--//我記憶里以前測試效率不錯,今天做具體探究.
1.建立測試環境:
$ mount | column | grep u01
/dev/cciss/c0d0p6 on /u01 type ext3 (rw)
--// /u01的文件系統是ext3.
--//按照順序建立文件。
$ cd /u01/testrm
$ time for i in $(seq -f "%06g" 300000); do >| $i.aud; done
$ time perl -e 'unlink for ( <*.aud> )'
real 0m4.077s
user 0m0.445s
sys 0m2.961s
2.rsync測試:
$ mkdir /u01/blanktest
--//建立一個空目錄.
$ cd /u01/testrm
$ time for i in $(seq -f "%06g" 300000); do >| $i.aud; done
real 0m7.174s
user 0m2.179s
sys 0m4.407s
$ time rsync -a --delete /u01/blanktest /u01/testrm
real 0m0.048s
user 0m0.003s
sys 0m0.003s
--//奇怪不可能這樣快,發現/u01/blanktest漏寫一個/.
$ time rsync -a --delete /u01/blanktest/ /u01/testrm
real 0m4.889s
user 0m0.360s
sys 0m3.831s
--//確實非常快,與perl的測試基本接近。
3.使用strace跟蹤看看.
$ time for i in $(seq -f "%06g" 300000); do >| $i.aud; done
real 0m6.508s
user 0m2.272s
sys 0m3.614s
$ ls -f | head
.
..
026879.aud
078531.aud
084876.aud
034744.aud
280025.aud
063529.aud
137869.aud
163787.aud
--//顯示順序與前面的測試一樣!!
$ strace -T -tt -o /tmp/yy4.txt rsync -a --delete /u01/blanktest/ /u01/testrm
--//你根本看不到unlink操作。
--//跟蹤內容如下:
....
16:01:20.455672 fcntl(3, F_SETFD, FD_CLOEXEC) = 0 <0.000015>
16:01:20.455739 getdents(3, /* 3 entries */, 32768) = 80 <0.000026>
16:01:20.455824 lstat("000001.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000018>
16:01:20.455924 mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6ebf790000 <0.000017>
16:01:20.456008 getdents(3, /* 0 entries */, 32768) = 0 <0.000016>
16:01:20.456070 close(3) = 0 <0.000018>
16:01:20.456144 mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6ebf74f000 <0.000016>
16:01:20.456216 select(5, NULL, [4], [4], {60, 0}) = 1 (out [4], left {59, 999997}) <0.000018>
16:01:20.456306 write(4, "=\0\0\7\5\f\1.\0\0\20e&*p\355A\0\0\201\366\6oracle\201\366\10o"..., 65) = 65 <0.000617>
16:01:20.457033 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {54, 864503}) <5.135527>
16:01:25.592682 read(5, "\3\0\0\7", 4) = 4 <0.000020>
16:01:25.622656 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999997}) <0.000015>
16:01:25.622717 read(5, "\1\10\0", 3) = 3 <0.000016>
16:01:25.622776 select(5, NULL, [4], [4], {60, 0}) = 1 (out [4], left {59, 999997}) <0.000013>
16:01:25.622824 write(4, "\3\0\0\7\1\10\0", 7) = 7 <0.000016>
16:01:25.622870 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999996}) <0.000026>
16:01:25.622978 read(5, "\23\0\0\7", 4) = 4 <0.000024>
16:01:25.623065 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999997}) <0.000025>
16:01:25.623159 read(5, "\2\10\200\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 19) = 19 <0.000025>
16:01:25.623260 open("000001.aud", O_RDONLY) = 3 <0.000049>
16:01:25.623354 fstat(3, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000011>
16:01:25.623470 close(3) = 0 <0.000013>
16:01:25.623512 select(5, NULL, [4], [4], {60, 0}) = 1 (out [4], left {59, 999995}) <0.000037>
16:01:25.623604 write(4, "'\0\0\7\2\10\200\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\324\35\214\331\217"..., 43) = 43 <0.000027>
16:01:25.623704 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999753}) <0.000262>
16:01:25.624019 read(5, "\1\0\0\7", 4) = 4 <0.000035>
16:01:25.624108 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999997}) <0.000012>
16:01:25.624167 read(5, "\0", 1) = 1 <0.000023>
16:01:25.624257 munmap(0x7f6ebf790000, 266240) = 0 <0.000023>
16:01:25.624323 munmap(0x7f6ebf7d1000, 266240) = 0 <0.000026>
16:01:25.624415 select(5, NULL, [4], [4], {60, 0}) = 1 (out [4], left {59, 999996}) <0.000024>
16:01:25.624509 write(4, "\1\0\0\7\0", 5) = 5 <0.000029>
16:01:25.624597 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999931}) <0.000083>
16:01:25.624731 read(5, "\2\0\0\7", 4) = 4 <0.000011>
16:01:25.624772 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999998}) <0.000025>
16:01:25.624850 read(5, "\0\0", 2) = 2 <0.000011>
16:01:25.624909 select(5, NULL, [4], [4], {60, 0}) = 1 (out [4], left {59, 999999}) <0.000010>
16:01:25.624970 write(4, "\1\0\0\7\0", 5) = 5 <0.000012>
16:01:25.625019 select(5, NULL, [4], [4], {60, 0}) = 1 (out [4], left {59, 999998}) <0.000010>
16:01:25.625060 write(4, "\1\0\0\7\0", 5) = 5 <0.000013>
16:01:25.625106 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999999}) <0.000013>
16:01:25.625194 read(5, "\1\0\0\7", 4) = 4 <0.000023>
16:01:25.625279 select(6, [5], [], NULL, {60, 0}) = 1 (in [5], left {59, 999998}) <0.000012>
16:01:25.625378 read(5, "\0", 1) = 1 <0.000032>
16:01:25.625462 wait4(4794, 0x7fff4321847c, WNOHANG, NULL) = 0 <0.000022>
16:01:25.625543 select(0, NULL, NULL, NULL, {0, 20000}) = 0 (Timeout) <0.020087>
16:01:25.645697 wait4(4794, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], WNOHANG, NULL) = 4794 <0.000027>
16:01:25.645787 --- SIGCHLD (Child exited) @ 0 (0) ---
16:01:25.645833 wait4(-1, 0x7fff43217e6c, WNOHANG, NULL) = -1 ECHILD (No child processes) <0.000018>
16:01:25.645900 rt_sigreturn(0xffffffffffffffff) = 4794 <0.000017>
16:01:25.645958 rt_sigaction(SIGUSR1, {0x1, [], SA_RESTORER, 0x3799030330}, NULL, 8) = 0 <0.000022>
16:01:25.646064 rt_sigaction(SIGUSR2, {0x1, [], SA_RESTORER, 0x3799030330}, NULL, 8) = 0 <0.000021>
16:01:25.646149 wait4(4794, 0x7fff43218474, WNOHANG, NULL) = -1 ECHILD (No child processes) <0.000020>
16:01:25.646267 exit_group(0) = ?
--//我不知道句柄5,6表示怎麼。發現rync實際上建立子進程來執行刪除操作。重新測試,strace加入-f參數。
--//補充文件大小不為0看看。
$ time for i in $(seq -f "%06g" 300000); do echo test $i >| $i.aud; done
real 0m44.288s
user 0m3.335s
sys 0m7.946s
$ time rsync -a --delete /u01/blanktest/ /u01/testrm
real 0m31.823s
user 0m0.351s
sys 0m4.612s
--//繼續strace的測試:
$ time for i in $(seq -f "%06g" 300000); do >| $i.aud; done
real 0m6.989s
user 0m2.238s
sys 0m4.112s
$ strace -f -T -tt -o /tmp/yy5.txt rsync -a --delete /u01/blanktest/ /u01/testrm
$ awk '{print $1}' /tmp/yy5.txt | sort | uniq -c
325 5158
600728 5159
72 5160
--//分成3個進程,主要有pid=5159完成。
--//查看跟蹤文件:
5159 16:15:08.681724 getdents(0, /* 1024 entries */, 32768) = 32752 <0.001046>
--//開始讀取目錄
5159 16:15:08.686451 lstat("026879.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000024>
5159 16:15:08.686571 mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f413480e000 <0.000021>
5159 16:15:08.686664 mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f41347cd000 <0.000017>
5159 16:15:08.686738 lstat("078531.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000020>
5159 16:15:08.686830 lstat("084876.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000020>
5159 16:15:08.686921 lstat("034744.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.687009 lstat("280025.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000020>
5159 16:15:08.687099 lstat("063529.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.687188 lstat("137869.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.687277 lstat("163787.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.687378 lstat("111415.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000021>
5159 16:15:08.687468 lstat("177021.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000020>
--//與前面perl執行看到非常相似。
--//你可以發現rsync的特點是getdents之後跟著lstat,註意看lstat裡面的文件與前面ls -f順序一致.
...
5159 16:15:08.778996 getdents(0, /* 1024 entries */, 32768) = 32768 <0.000950>
5159 16:15:08.783589 lstat("121204.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000020>
5159 16:15:08.783689 lstat("006462.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000018>
5159 16:15:08.783782 lstat("235760.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.783871 lstat("187943.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.783959 lstat("235116.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000018>
5159 16:15:08.784047 lstat("077735.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000018>
5159 16:15:08.784134 lstat("126397.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.784223 lstat("098220.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.784319 lstat("209702.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
5159 16:15:08.784410 lstat("277317.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000018>
5159 16:15:08.784497 lstat("269846.aud", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0 <0.000019>
...
--//接著getdents之後跟著lstat,如此反覆.
5159 16:15:37.270786 getdents(0, /* 0 entries */, 32768) = 0 <0.000017>
5159 16:15:37.270854 close(0) = 0 <0.000030>
5159 16:15:37.270938 mmap(NULL, 1200128, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f41331fe000 <0.000019>
5159 16:15:37.537610 munmap(0x7f41331fe000, 1200128) = 0 <0.000140>
5159 16:15:37.552144 unlink("300000.aud") = 0 <0.000098>
5159 16:15:37.552364 unlink("299999.aud") = 0 <0.000060>
5159 16:15:37.552466 unlink("299998.aud") = 0 <0.000023>
5159 16:15:37.552512 unlink("299997.aud") = 0 <0.000051>
5159 16:15:37.552608 unlink("299996.aud") = 0 <0.000038>
5159 16:15:37.552685 unlink("299995.aud") = 0 <0.000051>
5159 16:15:37.552769 unlink("299994.aud") = 0 <0.000035>
5159 16:15:37.552843 unlink("299993.aud") = 0 <0.000054>
5159 16:15:37.552930 unlink("299992.aud") = 0 <0.000020>
--//最後集中做unlink操作,註意看unlink的文件按照文件名反序排序刪除的,這樣刪除文件最快,這也是prsync與perl一樣,測試刪除
--//文件快的原因。
--//只要按照文件名順序建立或者反序建立做刪除操作,perl或者rsync刪除文件都是最快的。
--//補充實際上按照inode的順序刪除最快(反向類似)。
4.亂序建立文件呢?
--//按照亂序建立文件,我不知道bash shell如何實現,我先通過oracle的sql語句建立亂序數字輸出。
set pagesize 0
spool aa.txt
select lpad(level,6,'0') from dual connect by level <=3e5 order by dbms_random.value;
spool off
--//註意設置set pagesize 0,不然中間出現多次頁頭。註意保持aa.txt的備份,測試要重覆多次。
$ time for i in $(cat /tmp/aa.txt ); do >| $i.aud; done
real 0m6.880s
user 0m1.986s
sys 0m4.282s
$ ls -f|head
.
..
026879.aud
078531.aud
084876.aud
034744.aud
280025.aud
063529.aud
137869.aud
163787.aud
--//顯示順序與前面的測試一樣!!
$ time rsync -a --delete /u01/blanktest/ /u01/testrm
real 1m53.208s
user 0m0.459s
sys 0m7.391s
--//差不多2分鐘,使用strace跟蹤可以看到前面unlink的順序,可以推斷,unlink按照文件名從大到小的順序刪除,這樣inode的順序是打亂的,rsync也不會很快
--//刪除全部文件,再次驗證我前面的判斷:按照inode的順序(不管正序還是反序),做刪除操作效率最高。
$ strace -f -T -tt -o /tmp/yy6.txt rsync -a --delete /u01/blanktest/ /u01/testrm
--//跟蹤文件內容如下:
9769 08:41:59.387415 getdents(0, /* 0 entries */, 32768) = 0 <0.000025>
9769 08:41:59.387503 close(0) = 0 <0.000036>
9769 08:41:59.387608 mmap(NULL, 1200128, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa39724d000 <0.000024>
9769 08:41:59.628093 munmap(0x7fa39724d000, 1200128) = 0 <0.000174>
9769 08:41:59.643044 unlink("300000.aud") = 0 <0.000086>
9769 08:41:59.643229 unlink("299999.aud") = 0 <0.000084>
9769 08:41:59.643361 unlink("299998.aud") = 0 <0.000085>
9769 08:41:59.643476 unlink("299997.aud") = 0 <0.000066>
9769 08:41:59.643588 unlink("299996.aud") = 0 <0.000029>
...
$ time for i in $(cat /tmp/aa.txt ); do >| $i.aud; done
real 0m6.881s
user 0m1.965s
sys 0m4.312s
$ time sed "s/$/.aud/" /tmp/aa.txt | xargs rm -f
real 0m6.191s
user 0m0.552s
sys 0m5.403s
$ time for i in $(cat /tmp/aa.txt ); do >| $i.aud; done
real 0m6.889s
user 0m1.990s
sys 0m4.280s
$ time sed "s/$/.aud/" /tmp/aa.txt|tac | xargs rm -f
real 0m7.161s
user 0m0.459s
sys 0m5.969s
--//反序始終慢一點,也許執行tac的原因。
--//按照inode的順序(不管正序還是反序),做刪除操作效率最高。
5.總結:
--//可以得出結論,要快速刪除某個目錄的大量文件,最快的方式是按照inode的順序執行刪除操作,這樣最快.