这篇文章主要为大家展示了“java如何实现对Hadoop进行操作”，内容简而易懂，条理清晰，希望能够帮助大家解决疑惑，下面让小编带领大家一起研究并学习一下“java如何实现对Hadoop进行操作”这篇文章吧。
基本操作
import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.*;import org.junit.Test;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.DisplayName;import org.junit.runner.RunWith;import org.junit.runners.JUnit4;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import java.util.Arrays;@RunWith(JUnit4.class)@DisplayName("Test using junit4")public class HadoopClientTest { private FileSystem fileSystem = null; @BeforeEach public void init() throws URISyntaxException, IOException, InterruptedException { Configuration configuration = new Configuration(); configuration.set("dfs.replication", "1"); configuration.set("dfs.blocksize", "64m"); fileSystem = FileSystem.get(new URI("hdfs://hd-even-01:9000"), configuration, "root"); } @Test public void copyFileFromLocal() throws URISyntaxException, IOException, InterruptedException { // 上传文件 fileSystem.copyFromLocalFile(new Path("C:\\Users\\Administrator\\Desktop\\win10激活.txt"), new Path("/even1")); // 关闭流，报错winUtils,因为使用了linux的tar包，如果windows要使用，则需要编译好这个winUtils包才能使用 fileSystem.close(); } @Test public void copyFileToLocal() throws URISyntaxException, IOException, InterruptedException { // 下载文件 fileSystem.copyToLocalFile(new Path("/win10激活.txt"), new Path("E:/")); // 关闭流，报错winUtils,因为使用了linux的tar包，如果windows要使用，则需要编译好这个winUtils包才能使用 fileSystem.close(); } @Test public void hdfsMkdir() throws IOException { // 调用创建文件夹方法 fileSystem.mkdirs(new Path("/even1")); // 关闭方法 fileSystem.close(); } public void hdfsRename() throws IOException { fileSystem.rename(new Path(""), new Path("")); fileSystem.close(); } @Test public void hdfsRm() throws IOException {// fileSystem.delete(new Path("")); // 第二个参数表示递归删除 fileSystem.delete(new Path(""), true); fileSystem.close(); } @Test public void hdfsLs() throws IOException { // 调用方法返回远程迭代器，第二个参数是把目录文件夹内的文件也列出来 RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true); while (listFiles.hasNext()) { LocatedFileStatus locatedFileStatus = listFiles.next(); System.out.println("文件路径：" + locatedFileStatus.getPath()); System.out.println("块大小：" + locatedFileStatus.getBlockSize()); System.out.println("文件长度：" + locatedFileStatus.getLen()); System.out.println("副本数量：" + locatedFileStatus.getReplication()); System.out.println("块信息：" + Arrays.toString(locatedFileStatus.getBlockLocations())); } fileSystem.close(); } @Test public void findHdfs() throws IOException { // 1，展示状态信息 FileStatus[] listStatus = fileSystem.listStatus(new Path("/")); // 2,遍历所有文件 for (FileStatus fileStatus : listStatus) { if (fileStatus.isFile()) System.out.println("是文件：" + fileStatus.getPath().getName()); else if (fileStatus.isDirectory()) System.out.println("是文件夹：" + fileStatus.getPath().getName()); } fileSystem.close(); }}文件读写
import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.junit.Before;import org.junit.Test;import org.junit.jupiter.api.DisplayName;import org.junit.runner.RunWith;import org.junit.runners.JUnit4;import java.io.*;import java.net.URI;import java.net.URISyntaxException;import java.nio.charset.StandardCharsets;import java.util.Arrays;@RunWith(JUnit4.class)@DisplayName("this is read write test!")public class HadoopReadWriteTest { FileSystem fileSystem = null; Configuration configuration = null; @Before public void init() throws URISyntaxException, IOException, InterruptedException { // 1，加载配置 configuration = new Configuration(); // 2，构建客户端 fileSystem = FileSystem.get(new URI("hdfs://hd-even-01:9000/"), configuration, "root"); } @Test public void testReadData() throws IOException { // 1，获取hdfs文件流 FSDataInputStream open = fileSystem.open(new Path("/win10激活.txt")); // 2，设置一次获取的大小 byte[] bytes = new byte[1024]; // 3，读取数据 while (open.read(bytes) != -1) System.out.println(Arrays.toString(bytes)); open.close(); fileSystem.close(); } @Test public void testReadData1() throws IOException { FSDataInputStream open = fileSystem.open(new Path("/win10激活.txt")); // 使用缓冲流会快点 BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(open, StandardCharsets.UTF_8)); String line = ""; while ((line = bufferedReader.readLine()) != null) { System.out.println(line); } bufferedReader.close(); open.close(); fileSystem.close(); } @Test public void readSomeData() throws IOException { FSDataInputStream open = fileSystem.open(new Path("/win10激活.txt")); // 指定开始的index open.seek(14); // 指定读的多少 byte[] bytes = new byte[5]; while (open.read(bytes) != -1) System.out.println(new String(bytes)); open.close(); fileSystem.close(); } @Test public void writeData() throws IOException { // 1，获取输出流 FSDataOutputStream out = fileSystem.create(new Path("/win11.txt"), false); // 2，获取需要写的文件输入流 FileInputStream in = new FileInputStream(new File("C:\\Users\\Administrator\\Desktop\\xixi.txt")); byte[] b = new byte[1024]; int read = 0; while ((read = in.read(b)) != -1) { out.write(b, 0, read); } in.close(); out.close(); fileSystem.close(); } @Test public void writeData1() throws IOException { // 1，创建输出流 FSDataOutputStream out = fileSystem.create(new Path("/aibaobao.txt"), false); // 2，写数据 out.write("wochaoaibaobao".getBytes()); // 3,关闭流 IOUtils.closeStream(out); fileSystem.close(); } @Test public void putToHdfs() throws IOException { // 1，获取输入流 FileInputStream in = new FileInputStream(new File("C:\\Users\\Administrator\\Desktop\\xixi.txt")); // 2，获取输出流 FSDataOutputStream out = fileSystem.create(new Path("/haddopPut.txt"), false); // 3，拷贝 IOUtils.copyBytes(in, out, configuration); // 4，关闭流 IOUtils.closeStream(in); IOUtils.closeStream(out); fileSystem.close(); } @Test public void getFromHdfs() throws IOException { // 1,获取输入流 FSDataInputStream open = fileSystem.open(new Path("/haddopPut.txt")); // 2,获取输出流 FileOutputStream out = new FileOutputStream(new File("C:\\Users\\Administrator\\Desktop\\haddopPut.txt")); // 3，拷贝 IOUtils.copyBytes(open, out, configuration); // 4，关闭流 IOUtils.closeStream(open); IOUtils.closeStream(out); fileSystem.close(); }}以上是“java如何实现对Hadoop进行操作”这篇文章的所有内容，感谢各位的阅读！相信大家都有了一定的了解，希望分享的内容对大家有所帮助，如果还想学习更多知识，欢迎关注编程网行业资讯频道！
