HDFS基礎

1. HDFS Shell基礎

[root@master hadoop]# hadoop fs
Usage: hadoop fs [generic options]
[-appendToFile <localsrc> ... <dst>]
[-cat [-ignoreCrc] <src> ...]
[-checksum <src> ...]
[-chgrp [-R] GROUP PATH...]
[-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...]
[-chown [-R] [OWNER][:[GROUP]] PATH...]
[-copyFromLocal [-f] [-p] [-l] <localsrc> ... <dst>]
[-copyToLocal [-p] [-ignoreCrc] [-crc] <src> ... <localdst>]
[-count [-q] [-h] <path> ...]
[-cp [-f] [-p | -p[topax]] <src> ... <dst>]
[-createSnapshot <snapshotDir> [<snapshotName>]]
[-deleteSnapshot <snapshotDir> <snapshotName>]
[-df [-h] [<path> ...]]
[-du [-s] [-h] <path> ...]
[-expunge]
[-find <path> ... <expression> ...]
[-get [-p] [-ignoreCrc] [-crc] <src> ... <localdst>]
[-getfacl [-R] <path>]
[-getfattr [-R] {-n name | -d} [-e en] <path>]
[-getmerge [-nl] <src> <localdst>]
[-help [cmd ...]]
[-ls [-d] [-h] [-R] [<path> ...]]
[-mkdir [-p] <path> ...]
[-moveFromLocal <localsrc> ... <dst>]
[-moveToLocal <src> <localdst>]
[-mv <src> ... <dst>]
[-put [-f] [-p] [-l] <localsrc> ... <dst>]
[-renameSnapshot <snapshotDir> <oldName> <newName>]
[-rm [-f] [-r|-R] [-skipTrash] <src> ...]
[-rmdir [--ignore-fail-on-non-empty] <dir> ...]
[-setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>]]
[-setfattr {-n name [-v value] | -x name} <path>]
[-setrep [-R] [-w] <rep> <path> ...]
[-stat [format] <path> ...]
[-tail [-f] <file>]
[-test -[defsz] <path>]
[-text [-ignoreCrc] <src> ...]
[-touchz <path> ...]
[-truncate [-w] <length> <path> ...]
[-usage [cmd ...]]

查看某個命令的具體幫助信息：

[root@master hadoop]# hadoop fs -help test
-test -[defsz] <path> :
Answer various questions about <path>, with result via exit status.
-d return 0 if <path> is a directory.
-e return 0 if <path> exists.
-f return 0 if <path> is a file.
-s return 0 if file <path> is greater than zero bytes in size.
-z return 0 if file <path> is zero bytes in size, else return 1.

這裡尤其要註意，如果文件存在，返回結果是0

我們上傳一個文件，然後用test命令測試：

[root@master hadoop]# hadoop fs -put /root/test test.txt

[root@master hadoop]# hadoop fs -ls .
Found 4 items
drwxrwxrwx - hdfs hdfs 0 2018-02-10 22:22 QuasiMonteCarlo_1518319340789_698036166
drwxrwxrwx - hdfs hdfs 0 2018-02-10 23:21 QuasiMonteCarlo_1518322909671_1083050937
-rw-r--r-- 3 root hdfs 5 2019-02-08 21:52 test
-rw-r--r-- 3 root hdfs 5 2019-02-09 02:12 test.txt

[root@master hadoop]# hadoop fs -test -e test.txt
[root@master hadoop]# echo $?
0
[root@master hadoop]# hadoop fs -test -e /user/root/test.txt
[root@master hadoop]# echo $?
0

[root@master hadoop]# hadoop fs -test -e /user/root/test.txtt
[root@master hadoop]# echo $?
1

2. HDFS 編程基礎

2.1 判斷文件是否存在

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSFileIfExist {
    public static void main(String[] args){
        try{
            //hdfs路徑
            String fileName = "/user/root/test.txt";
            Configuration conf = new Configuration();
            conf.set("fs.defaultFS", "hdfs://master:8020");
            conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
            FileSystem fs = FileSystem.get(conf);
            if(fs.exists(new Path(fileName))){
                System.out.println("文件存在");
            }else{
                System.out.println("文件不存在");
            }
            
        }catch (Exception e){
            e.printStackTrace();
        }
    }

}

上面的代碼中fs.defaultFS的值，請查看hdfs配置文件 core-site.xml ，該文件位於hadoop安裝目錄的etc/hadoop目錄下。

如果在Windows系統中運行該程式，請在C:\Windows\System32\drivers\etc\hosts文件中加上主機名master和ip地址的對應關係。

2.2 列出目錄下的文件及子目錄

FileStatus[] list = fs.listStatus(new Path("/user/root/"));
//System.out.println("==========================================================");
if(list != null)
for (FileStatus f : list) {
     System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath().getName(), f.isDirectory(), f.getLen());
}