文章详情

短信预约-IT技能 免费直播动态提醒

请输入下面的图形验证码

提交验证

短信预约提醒成功

Java之Rsync并发迁移数据并校验详解

2024-04-02 19:55

关注

java调用Rsync并发迁移数据并执行校验

java代码如下

RsyncFile.java


import lombok.NoArgsConstructor;
import lombok.SneakyThrows;

import java.io.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.concurrent.*;


@NoArgsConstructor
public class RsyncFile implements  Runnable{
    private static final int availProcessors = Runtime.getRuntime().availableProcessors();
    //构造以cpu核心数为核心池,cpu线程数为最大池,超时时间为1s,线程队列为大小为无界的安全阻塞线程队列,拒绝策略为DiscardOldestPolicy()的线程池。(同步数据当然不能丢下拒绝任务)
    private ExecutorService ThreadPool = new ThreadPoolExecutor(availProcessors >> 1,
            availProcessors,
            1L,
            TimeUnit.SECONDS,
            new LinkedBlockingQueue<>(),Executors.defaultThreadFactory(),
            new ThreadPoolExecutor.DiscardOldestPolicy());

    //保存扫描得到的文件列表
    private static ArrayList<String> fileNameList = new ArrayList<String>();
    private String shellname;
    private String filename;
    private String userip;
    private CountDownLatch countDownLatch;
    private static int deep = 0;

    public RsyncFile(String ShellName, String filename, String UserIP, CountDownLatch countDownLatch) {
        this.shellname = ShellName;
        this.filename = filename;
        this.userip = UserIP;
        this.countDownLatch = countDownLatch;
    }

    public static void main(String[] args) {
        try {
            new RsyncFile().Do(args[0],args[1],Integer.parseInt(args[2]));
        }catch (ArrayIndexOutOfBoundsException e){
            System.out.println(e);
            System.out.println("Error , args send fault");
            System.out.println("please send localAddress remote username @ remote IP or hostname and catalogue");
            System.out.println("like this [  /home/test/  root@node1:/test/   1 ]");
        }catch(NumberFormatException e1){
            System.out.println(e1);
            System.out.println("please input Right Directory depth, this number must be int");
            System.out.println("like this [  /home/test/  root@node1:/test/   1 ]");
        }
    }

    @SneakyThrows
    private void Do(String content,String UserIP,int setdeep){
        System.out.println("开始执行");
        System.out.println("开始时间:" + new Date());
        Long a = System.nanoTime();
        File file = new File(content);
        System.out.println("开始扫描本地指定目录");
        GetAllFile(file,setdeep);//按深度扫描非空文件夹和文件
        System.out.println("扫描本地目录完成");

        //给脚本赋予权限
        String [] cmd={"/bin/sh","-c","chmod 755 ./do*"};
        Runtime.getRuntime().exec(cmd);
        //创建远端目录操作
        System.out.println("开始创建远端目录结构");
        //一次计数锁用于保证目录创建完成
        CountDownLatch doDirLock = new CountDownLatch(1);
        ThreadPool.execute(new RsyncFile("./doDirc.sh",content,UserIP,doDirLock));
        doDirLock.await();
        System.out.println("创建远端目录结构完成");

        //开始同步工作
        System.out.println("开始执行同步工作");
        System.out.println("同步的文件夹或文件总数: " + fileNameList.size());
        System.out.println("正在同步。。。。。");
        //fileNameList.size()次计数锁用于保证数据同步完成(保证计时准确)
        CountDownLatch rsyncLock = new CountDownLatch(fileNameList.size());
        System.out.println(fileNameList.size());
        for (String fileName:fileNameList) {
            //除去文件名中与UserIP重复的文件路径
            String RemoteDir = UserIP.concat(fileName.replace(content, ""));
            System.out.println("要同步的本地目录或文件: " + fileName);
            System.out.println("要同步的远端目录或文件: " + RemoteDir);
            ThreadPool.execute(new RsyncFile("./doRync.sh",fileName, RemoteDir,rsyncLock));
        }
        rsyncLock.await();
        System.out.println("执行同步工作完成");

        //开始文件校验工作
        System.out.println("执行文件校验及重传");
        //一次计数锁用于保证校验完成
        CountDownLatch chechSumLock = new CountDownLatch(1);
        ThreadPool.execute(new RsyncFile("./doChecksum.sh",content,UserIP,chechSumLock));
        chechSumLock.await();
        System.out.println("文件校验及重传完成");
        ThreadPool.shutdown();

        Long b = System.nanoTime();
        Long aLong = (b - a)/1000000L;
        System.out.println("处理时间" + aLong + "ms");
        System.out.println("结束时间:" + new Date());

    }

    
    @Override
    public void run() {
        try {
            String command=shellname.concat(" ").concat(filename).concat(" ").concat(userip);
            File wd = new File("/bin");
            Process process = null;
            process = Runtime.getRuntime().exec("/bin/bash", null, wd);
            if (process != null) {
                InputStream is = process.getInputStream();
                BufferedReader reader = new BufferedReader(new InputStreamReader(is));
                PrintWriter out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(process.getOutputStream())),
                        true);
                //切换到当前class文件所在目录
                out.println("cd " + System.getProperty("user.dir"));
                out.println(command);
                out.println("exit");
                StringBuilder sb = new StringBuilder();
                String line;
                while ((line = reader.readLine()) != null) {
                    sb.append(line + System.lineSeparator());
                }
                process.waitFor();
                reader.close();
                out.close();
                process.destroy();
                System.out.println("result:" + sb.toString());
            }else {
                System.out.println("找不到系统bash工具,请检查系统是否异常,并为系统创建/bin/sh的bash工具软连接");
            }
        } catch (Exception e) {
            System.err.println(e.getMessage());
        }finally {
            //倒记数锁释放一次
            countDownLatch.countDown();
        }
    }


    
    @SneakyThrows
    private static void  GetAllFile(File file, int setDeep) {
        if(file != null){
            if(file.isDirectory() && deep<setDeep){
                deep++;
                File f[] = file.listFiles();
                if(f != null) {
                    int length = f.length;
                    for(int i = 0; i < length; i++)
                        GetAllFile(f[i],setDeep);
                    deep--;
                }
            } else {
                if(file.isDirectory()){
                    //如果为目录末尾添加 / 保证rsync正常处理
                    fileNameList.add(file.getAbsolutePath().concat("/"));
                }else {
                    fileNameList.add(file.getAbsolutePath());
                }
            }
        }
    }

}

doDir.sh


rsync -av --include='*/' --exclude='*' $1 $2 |tee -a /tmp/rsync.log 2>&1
echo "创建目录结构操作"

doRsync.sh


rsync -avzi --stats --progress $1 $2 |tee -a /tmp/rsync.log 2>&1

doChecksum.sh


rsync -acvzi --stats --progress $1 $2 |tee -a /tmp/checksum.log 2>&1

附录

rsync输出日志说明如下


YXcstpoguax  path/to/file
|||||||||||
||||||||||╰- x: The extended attribute information changed
|||||||||╰-- a: The ACL information changed
||||||||╰--- u: The u slot is reserved for future use
|||||||╰---- g: Group is different
||||||╰----- o: Owner is different
|||||╰------ p: Permission are different
||||╰------- t: Modification time is different
|||╰-------- s: Size is different
||╰--------- c: Different checksum (for regular files), or
||              changed value (for symlinks, devices, and special files)
|╰---------- the file type:
|            f: for a file,
|            d: for a directory,
|            L: for a symlink,
|            D: for a device,
|            S: for a special file (e.g. named sockets and fifos)
╰----------- the type of update being done::
             <: file is being transferred to the remote host (sent)
             >: file is being transferred to the local host (received)
             c: local change/creation for the item, such as:
                - the creation of a directory
                - the changing of a symlink,
                - etc.
             h: the item is a hard link to another item (requires 
                --hard-links).
             .: the item is not being updated (though it might have
                attributes that are being modified)
             *: means that the rest of the itemized-output area contains
                a message (e.g. "deleting")

到此这篇关于Java之Rsync并发迁移数据并校验详解的文章就介绍到这了,更多相关Java之Rsync并发内容请搜索编程网以前的文章或继续浏览下面的相关文章希望大家以后多多支持编程网!

阅读原文内容投诉

免责声明:

① 本站未注明“稿件来源”的信息均来自网络整理。其文字、图片和音视频稿件的所属权归原作者所有。本站收集整理出于非商业性的教育和科研之目的,并不意味着本站赞同其观点或证实其内容的真实性。仅作为临时的测试数据,供内部测试之用。本站并未授权任何人以任何方式主动获取本站任何信息。

② 本站未注明“稿件来源”的临时测试数据将在测试完成后最终做删除处理。有问题或投稿请发送至: 邮箱/279061341@qq.com QQ/279061341

软考中级精品资料免费领

  • 历年真题答案解析
  • 备考技巧名师总结
  • 高频考点精准押题
  • 2024年上半年信息系统项目管理师第二批次真题及答案解析(完整版)

    难度     813人已做
    查看
  • 【考后总结】2024年5月26日信息系统项目管理师第2批次考情分析

    难度     354人已做
    查看
  • 【考后总结】2024年5月25日信息系统项目管理师第1批次考情分析

    难度     318人已做
    查看
  • 2024年上半年软考高项第一、二批次真题考点汇总(完整版)

    难度     435人已做
    查看
  • 2024年上半年系统架构设计师考试综合知识真题

    难度     224人已做
    查看

相关文章

发现更多好内容

猜你喜欢

AI推送时光机
位置:首页-资讯-后端开发
咦!没有更多了?去看看其它编程学习网 内容吧
首页课程
资料下载
问答资讯