sftp 下载实现

使用零拷贝技术优化合并分块的功能
This commit is contained in:
laoyuyu
2020-01-15 20:28:10 +08:00
parent 6f53235807
commit 50b265f22a
37 changed files with 934 additions and 257 deletions

View File

@@ -15,21 +15,17 @@
*/
package com.arialyy.aria.sftp;
import android.text.TextUtils;
import com.arialyy.aria.core.FtpUrlEntity;
import com.arialyy.aria.core.IdEntity;
import com.arialyy.aria.core.loader.IInfoTask;
import com.arialyy.aria.core.loader.ILoaderVisitor;
import com.arialyy.aria.core.wrapper.AbsTaskWrapper;
import com.arialyy.aria.exception.BaseException;
import com.arialyy.aria.ftp.FtpTaskOption;
import com.arialyy.aria.util.CommonUtil;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
import java.io.UnsupportedEncodingException;
import java.util.Properties;
/**
* 进行登录获取session获取文件信息
@@ -51,10 +47,10 @@ public abstract class AbsSFtpInfoTask<WP extends AbsTaskWrapper> implements IInf
@Override public void run() {
try {
FtpUrlEntity entity = option.getUrlEntity();
String key = CommonUtil.getStrMd5(entity.hostName + entity.port + entity.user);
String key = CommonUtil.getStrMd5(entity.hostName + entity.port + entity.user + 0);
Session session = SFtpSessionManager.getInstance().getSession(key);
if (session == null) {
session = login(entity);
session = SFtpUtil.getInstance().getSession(entity, 0);
}
getFileInfo(session);
} catch (JSchException e) {
@@ -69,39 +65,6 @@ public abstract class AbsSFtpInfoTask<WP extends AbsTaskWrapper> implements IInf
}
}
private Session login(FtpUrlEntity entity) throws JSchException, UnsupportedEncodingException {
JSch jSch = new JSch();
IdEntity idEntity = entity.idEntity;
if (idEntity.prvKey != null) {
if (idEntity.pubKey == null) {
jSch.addIdentity(idEntity.prvKey,
entity.password == null ? null : entity.password.getBytes("UTF-8"));
} else {
jSch.addIdentity(idEntity.prvKey, idEntity.pubKey,
entity.password == null ? null : entity.password.getBytes("UTF-8"));
}
}
Session session;
if (TextUtils.isEmpty(entity.user)) {
session = jSch.getSession(entity.url, entity.hostName, Integer.parseInt(entity.port));
} else {
session = jSch.getSession(entity.hostName);
}
if (!TextUtils.isEmpty(entity.password)) {
session.setPassword(entity.password);
}
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);// 为Session对象设置properties
session.setTimeout(3000);// 设置超时
session.setIdentityRepository(jSch.getIdentityRepository());
session.connect();
return session;
}
protected FtpTaskOption getOption() {
return option;
}

View File

@@ -49,32 +49,33 @@ public class SFtpSessionManager {
/**
* 获取session获取完成session后检查map中的所有session移除所有失效的session
*
* @param key md5(host + port + userName )
* @param key md5(host + port + userName + threadId)
* @return 如果session不可用返回null
*/
public Session getSession(String key) {
if (TextUtils.isEmpty(key)) {
ALog.e(TAG, "获取session失败key为空");
ALog.e(TAG, "从缓存获取session失败key为空");
return null;
}
Session session = sessionDeque.get(key);
if (session == null) {
ALog.w(TAG, "获取session失败key" + key);
ALog.w(TAG, "从缓存获取session失败key" + key);
}
cleanIdleSession();
//cleanIdleSession();
return session;
}
/**
* 添加session
*/
public void addSession(Session session) {
public void addSession(Session session, int threadId) {
if (session == null) {
ALog.e(TAG, "添加session到管理器失败session 为空");
return;
}
String key =
CommonUtil.getStrMd5(session.getHost() + session.getPort() + session.getUserName());
CommonUtil.getStrMd5(
session.getHost() + session.getPort() + session.getUserName() + threadId);
sessionDeque.put(key, session);
}

View File

@@ -16,16 +16,19 @@
package com.arialyy.aria.sftp;
import android.text.TextUtils;
import com.arialyy.aria.util.ALog;
import com.arialyy.aria.core.FtpUrlEntity;
import com.arialyy.aria.core.IdEntity;
import com.arialyy.aria.util.CommonUtil;
import com.jcraft.jsch.ChannelExec;
import com.arialyy.aria.util.FileUtil;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import java.io.BufferedReader;
import com.jcraft.jsch.UserInfo;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.Properties;
/**
@@ -35,166 +38,137 @@ import java.util.Properties;
*/
public class SFtpUtil {
private final String TAG = CommonUtil.getClassName(getClass());
/**
* 用于执行命令
*/
public static final String CMD_TYPE_EXEC = "exec";
/**
* 用于处理文件
*/
public static final String CMD_TYPE_SFTP = "sftp";
private String ip, userName, password;
private int port;
private Session session;
private boolean isLogin = false;
private static SFtpUtil INSTANCE;
private SFtpUtil() {
createClient();
}
/**
* 创建客户端
*/
private void createClient() {
JSch jSch = new JSch();
try {
if (TextUtils.isEmpty(userName)) {
session = jSch.getSession(userName, ip, port);
} else {
session = jSch.getSession(ip);
public synchronized static SFtpUtil getInstance() {
if (INSTANCE == null) {
synchronized (SFtpUtil.class) {
INSTANCE = new SFtpUtil();
}
if (!TextUtils.isEmpty(password)) {
session.setPassword(password);
}
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);// 为Session对象设置properties
session.setTimeout(3000);// 设置超时
login();
isLogin = true;
} catch (JSchException e) {
e.printStackTrace();
}
return INSTANCE;
}
/**
* 执行登录
*/
public Session login() {
try {
session.connect(); // 通过Session建立连接
} catch (JSchException e) {
e.printStackTrace();
}
return session;
}
/**
* 登出
*/
public void logout() {
if (session != null) {
session.disconnect();
}
isLogin = false;
}
public Session getSession() {
return session;
}
/**
* 执行命令
* 创建jsch 的session
*
* @param cmd sftp命令
* @param threadId 线程id默认0
* @throws JSchException
* @throws UnsupportedEncodingException
*/
public void execCommand(String cmd) {
if (TextUtils.isEmpty(cmd)) {
ALog.e(TAG, "命令为空");
return;
}
if (!isLogin) {
ALog.e(TAG, "没有登录");
return;
}
ChannelExec channel = null;
try {
channel = (ChannelExec) session.openChannel(CMD_TYPE_EXEC);
channel.setCommand(cmd);
channel.connect();
String rst = getResult(channel.getInputStream());
public Session getSession(FtpUrlEntity entity, int threadId) throws JSchException,
UnsupportedEncodingException {
ALog.i(TAG, String.format("result: %s", rst));
JSch jSch = new JSch();
IdEntity idEntity = entity.idEntity;
if (idEntity.prvKey != null) {
if (idEntity.pubKey == null) {
jSch.addIdentity(idEntity.prvKey,
entity.password == null ? null : idEntity.prvPass.getBytes("UTF-8"));
} else {
jSch.addIdentity(idEntity.prvKey, idEntity.pubKey,
entity.password == null ? null : idEntity.prvPass.getBytes("UTF-8"));
}
}
setknowHost(jSch, entity);
Session session;
if (TextUtils.isEmpty(entity.user)) {
session = jSch.getSession(null, entity.hostName, Integer.parseInt(entity.port));
} else {
session = jSch.getSession(entity.user, entity.hostName, Integer.parseInt(entity.port));
}
if (!TextUtils.isEmpty(entity.password)) {
session.setPassword(entity.password);
}
Properties config = new Properties();
// 不检查公钥需要在connect之前配置但是不安全no 模式会自动将配对信息写入know_host文件
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);// 为Session对象设置properties
session.setTimeout(3000);// 设置超时
session.setIdentityRepository(jSch.getIdentityRepository());
session.connect();
SFtpSessionManager.getInstance().addSession(session, threadId);
return session;
}
private void setknowHost(JSch jSch, FtpUrlEntity entity) throws JSchException {
IdEntity idEntity = entity.idEntity;
if (idEntity.knowHost != null) {
File knowFile = new File(idEntity.knowHost);
if (!knowFile.exists()) {
FileUtil.createFile(knowFile);
}
jSch.setKnownHosts(idEntity.knowHost);
//HostKeyRepository hkr = jSch.getHostKeyRepository();
//hkr.add(new HostKey(entity.hostName, HostKey.SSHRSA, getPubKey(idEntity.pubKey)), new JschUserInfo());
//
//HostKey[] hks = hkr.getHostKey();
//if (hks != null) {
// System.out.println("Host keys in " + hkr.getKnownHostsRepositoryID());
// for (int i = 0; i < hks.length; i++) {
// HostKey hk = hks[i];
// System.out.println(hk.getHost() + " " +
// hk.getType() + " " +
// hk.getFingerPrint(jSch));
// }
//}
}
}
private byte[] getPubKey(String pubKeyPath) {
try {
File f = new File(pubKeyPath);
FileInputStream fis = new FileInputStream(f);
byte[] buf = new byte[(int) f.length()];
int len = fis.read(buf);
fis.close();
return buf;
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (JSchException e) {
e.printStackTrace();
} finally {
if (channel != null) {
channel.disconnect();
}
}
return null;
}
/**
* 执行命令后,获取服务器端返回的数据
*
* @return 服务器端返回的数据
*/
private String getResult(InputStream in) throws IOException {
if (in == null){
ALog.e(TAG, "输入流为空");
private static class JschUserInfo implements UserInfo {
@Override public String getPassphrase() {
return null;
}
StringBuilder sb = new StringBuilder();
BufferedReader isr = new BufferedReader(new InputStreamReader(in));
String line;
while ((line = isr.readLine()) != null) {
sb.append(line);
}
in.close();
isr.close();
return sb.toString();
}
public static class Builder {
private String ip, userName, password;
private int port = 22;
public Builder setIp(String ip) {
this.ip = ip;
return this;
@Override public String getPassword() {
return null;
}
public Builder setUserName(String userName) {
this.userName = userName;
return this;
@Override public boolean promptPassword(String message) {
System.out.println(message);
return true;
}
public Builder setPassword(String password) {
this.password = password;
return this;
@Override public boolean promptPassphrase(String message) {
System.out.println(message);
return false;
}
public Builder setPort(int port) {
this.port = port;
return this;
@Override public boolean promptYesNo(String message) {
System.out.println(message);
return false;
}
public SFtpUtil build() {
SFtpUtil login = new SFtpUtil();
login.ip = ip;
login.userName = userName;
login.password = password;
login.port = port;
if (TextUtils.isEmpty(ip)) {
throw new IllegalArgumentException("ip不能为空");
}
if (port < 0 || port > 65534) {
throw new IllegalArgumentException("端口错误");
}
return login;
@Override public void showMessage(String message) {
System.out.println(message);
}
}
}

View File

@@ -17,6 +17,7 @@ package com.arialyy.aria.sftp.download;
import com.arialyy.aria.core.common.CompleteInfo;
import com.arialyy.aria.core.download.DTaskWrapper;
import com.arialyy.aria.ftp.FtpTaskOption;
import com.arialyy.aria.sftp.AbsSFtpInfoTask;
import com.arialyy.aria.util.CommonUtil;
import com.jcraft.jsch.ChannelSftp;
@@ -37,13 +38,20 @@ final class SFtpDInfoTask extends AbsSFtpInfoTask<DTaskWrapper> {
@Override protected void getFileInfo(Session session) throws JSchException,
UnsupportedEncodingException, SftpException {
FtpTaskOption option = (FtpTaskOption) getWrapper().getTaskOption();
ChannelSftp channel = (ChannelSftp) session.openChannel("sftp");
SftpATTRS attr = channel.stat(
CommonUtil.convertFtpChar(getOption().getCharSet(), getWrapper().getEntity().getUrl()));
channel.connect(1000);
//channel.setFilenameEncoding(option.getCharSet());
//channel.setFilenameEncoding("gbk");
String remotePath = option.getUrlEntity().remotePath;
String temp = CommonUtil.convertFtpChar(getOption().getCharSet(), remotePath);
SftpATTRS attr = channel.stat(temp);
getWrapper().getEntity().setFileSize(attr.getSize());
CompleteInfo info = new CompleteInfo();
info.code = 200;
info.obj = channel;
channel.disconnect();
callback.onSucceed(getWrapper().getKey(), info);
}
}

View File

@@ -34,7 +34,6 @@ import com.arialyy.aria.core.task.IThreadTask;
import com.arialyy.aria.core.wrapper.AbsTaskWrapper;
import com.arialyy.aria.exception.BaseException;
import com.arialyy.aria.util.FileUtil;
import com.jcraft.jsch.ChannelSftp;
import java.io.File;
final class SFtpDLoader extends AbsNormalLoader {
@@ -42,7 +41,6 @@ final class SFtpDLoader extends AbsNormalLoader {
private int startThreadNum; //启动的线程数
private boolean isComplete = false;
private Looper looper;
private ChannelSftp channelSftp;
SFtpDLoader(AbsTaskWrapper wrapper, IEventListener listener) {
super(wrapper, listener);
@@ -103,9 +101,6 @@ final class SFtpDLoader extends AbsNormalLoader {
mStateManager.setLooper(mRecord, looper);
// 创建线程任务
SFtpDTTBuilderAdapter ttBuild =
(SFtpDTTBuilderAdapter) ((NormalTTBuilder) mTTBuilder).getAdapter();
ttBuild.setChannel(channelSftp);
getTaskList().addAll(mTTBuilder.buildThreadTask(mRecord,
new Handler(looper, mStateManager.getHandlerCallback())));
startThreadNum = mTTBuilder.getCreatedThreadNum();
@@ -143,7 +138,6 @@ final class SFtpDLoader extends AbsNormalLoader {
mInfoTask = infoTask;
infoTask.setCallback(new IInfoTask.Callback() {
@Override public void onSucceed(String key, CompleteInfo info) {
channelSftp = (ChannelSftp) info.obj;
startThreadTask();
}

View File

@@ -33,7 +33,7 @@ import com.arialyy.aria.ftp.download.FtpDRecordHandler;
*/
public class SFtpDLoaderUtil extends AbsNormalLoaderUtil {
protected SFtpDLoaderUtil(AbsTaskWrapper wrapper, IEventListener listener) {
public SFtpDLoaderUtil(AbsTaskWrapper wrapper, IEventListener listener) {
super(wrapper, listener);
wrapper.generateTaskOption(FtpTaskOption.class);
}
@@ -47,7 +47,8 @@ public class SFtpDLoaderUtil extends AbsNormalLoaderUtil {
structure.addComponent(new FtpDRecordHandler((DTaskWrapper) getTaskWrapper()))
.addComponent(new NormalThreadStateManager(getListener()))
.addComponent(new SFtpDInfoTask((DTaskWrapper) getTaskWrapper()))
.addComponent(new NormalTTBuilder(getTaskWrapper(), new SFtpDTTBuilderAdapter()git));
.addComponent(new NormalTTBuilder(getTaskWrapper(), new SFtpDTTBuilderAdapter(
(DTaskWrapper) getTaskWrapper())));
structure.accept(getLoader());
return structure;
}

View File

@@ -16,25 +16,30 @@
package com.arialyy.aria.sftp.download;
import android.os.Handler;
import com.arialyy.aria.core.FtpUrlEntity;
import com.arialyy.aria.core.TaskRecord;
import com.arialyy.aria.core.ThreadRecord;
import com.arialyy.aria.core.common.SubThreadConfig;
import com.arialyy.aria.core.download.DTaskWrapper;
import com.arialyy.aria.core.loader.AbsNormalTTBuilderAdapter;
import com.arialyy.aria.core.loader.IRecordHandler;
import com.arialyy.aria.core.task.IThreadTaskAdapter;
import com.arialyy.aria.ftp.FtpTaskOption;
import com.arialyy.aria.sftp.SFtpSessionManager;
import com.arialyy.aria.sftp.SFtpUtil;
import com.arialyy.aria.util.ALog;
import com.arialyy.aria.util.CommonUtil;
import com.arialyy.aria.util.FileUtil;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import java.io.File;
import java.io.UnsupportedEncodingException;
class SFtpDTTBuilderAdapter extends AbsNormalTTBuilderAdapter {
private ChannelSftp channel;
private FtpTaskOption option;
SFtpDTTBuilderAdapter() {
}
void setChannel(ChannelSftp channel) {
this.channel = channel;
SFtpDTTBuilderAdapter(DTaskWrapper wrapper) {
option = (FtpTaskOption) wrapper.getTaskOption();
}
@Override public IThreadTaskAdapter getAdapter(SubThreadConfig config) {
@@ -46,7 +51,21 @@ class SFtpDTTBuilderAdapter extends AbsNormalTTBuilderAdapter {
boolean isBlock, int startNum) {
SubThreadConfig config =
super.getSubThreadConfig(stateHandler, threadRecord, isBlock, startNum);
config.obj = channel;
FtpUrlEntity entity = option.getUrlEntity();
String key =
CommonUtil.getStrMd5(entity.hostName + entity.port + entity.user + threadRecord.threadId);
Session session = SFtpSessionManager.getInstance().getSession(key);
if (session == null) {
try {
session = SFtpUtil.getInstance().getSession(entity, threadRecord.threadId);
} catch (JSchException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
config.obj = session;
return config;
}

View File

@@ -17,6 +17,19 @@ package com.arialyy.aria.sftp.download;
import com.arialyy.aria.core.common.SubThreadConfig;
import com.arialyy.aria.core.task.AbsThreadTaskAdapter;
import com.arialyy.aria.exception.AriaException;
import com.arialyy.aria.ftp.FtpTaskOption;
import com.arialyy.aria.util.ALog;
import com.arialyy.aria.util.CommonUtil;
import com.arialyy.aria.util.FileUtil;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpProgressMonitor;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
/**
* sftp 线程任务适配器
@@ -24,12 +37,154 @@ import com.arialyy.aria.core.task.AbsThreadTaskAdapter;
* @author lyy
*/
final class SFtpDThreadTaskAdapter extends AbsThreadTaskAdapter {
private ChannelSftp channelSftp;
private Session session;
private FtpTaskOption option;
SFtpDThreadTaskAdapter(SubThreadConfig config) {
super(config);
session = (Session) config.obj;
option = (FtpTaskOption) getTaskWrapper().getTaskOption();
}
@Override protected void handlerThreadTask() {
if (session == null) {
fail(new AriaException(TAG, "session 为空"), false);
return;
}
FileOutputStream fos;
try {
int timeout = getTaskConfig().getConnectTimeOut();
if (!session.isConnected()) {
session.connect(timeout);
}
channelSftp = (ChannelSftp) session.openChannel("sftp");
channelSftp.connect(timeout);
fos = new FileOutputStream(getThreadConfig().tempFile, true);
if (channelSftp.isClosed() || !channelSftp.isConnected()) {
channelSftp.connect();
}
ALog.d(TAG,
String.format("任务【%s】线程__%s__开始下载【开始位置 : %s结束位置%s】", getTaskWrapper().getKey(),
getThreadRecord().threadId, getThreadRecord().startLocation,
getThreadRecord().endLocation));
// 开启服务器对UTF-8的支持如果服务器支持就用UTF-8编码
String charSet = option.getCharSet();
String remotePath =
CommonUtil.convertFtpChar(charSet, option.getUrlEntity().remotePath);
if (getThreadRecord().startLocation > 0) {
channelSftp.get(remotePath, fos, new Monitor(true), ChannelSftp.RESUME,
getThreadRecord().startLocation);
} else {
channelSftp.get(remotePath, fos, new Monitor(false));
}
} catch (Exception e) {
e.printStackTrace();
} finally {
channelSftp.disconnect();
}
}
private class Monitor implements SftpProgressMonitor {
private boolean isResume;
private Monitor(boolean isResume) {
this.isResume = isResume;
}
@Override public void init(int op, String src, String dest, long max) {
ALog.d(TAG, String.format("op = %s; src = %s; dest = %s; max = %s", op, src, dest, max));
}
/**
* @param count 已传输的数据
* @return false 取消任务
*/
@Override public boolean count(long count) {
if (mSpeedBandUtil != null) {
mSpeedBandUtil.limitNextBytes((int) count);
}
/*
* jsch 如果是恢复任务第一次回调count会将已下载的长度返回后面才是新增的文件长度。
* 所以恢复任务的话,需要忽略一次回调
*/
if (!isResume) {
progress(count);
}
isResume = false;
//return !getThreadTask().isBreak() && getRangeProgress() < getThreadRecord().endLocation;
if (getRangeProgress() > getThreadRecord().endLocation) {
return false;
}
return !getThreadTask().isBreak();
}
@Override public void end() {
if (getThreadTask().isBreak()) {
return;
}
complete();
//boolean isSuccess = true;
//// 剪裁文件
//if (getRangeProgress() > getThreadRecord().endLocation) {
// isSuccess = clipFile();
//}
//if (isSuccess) {
// complete();
//} else {
// fail(new AriaException(TAG, "剪切文件失败"), false);
//}
}
/**
* 文件超出内容,剪切文件
*
* @return true 剪切文件成功
*/
private boolean clipFile() {
FileInputStream fis = null;
FileOutputStream fos = null;
long stime = System.currentTimeMillis();
try {
String destPath = getThreadConfig().tempFile.getPath();
ALog.d(TAG, "oldSize = " + getThreadConfig().tempFile.length());
String tempPath = destPath + "_temp";
fis = new FileInputStream(getThreadConfig().tempFile);
fos = new FileOutputStream(tempPath);
FileChannel inChannel = fis.getChannel();
FileChannel outChannel = fos.getChannel();
inChannel.transferTo(0, getThreadRecord().endLocation, outChannel);
FileUtil.deleteFile(getThreadConfig().tempFile);
File oldF = new File(tempPath);
File newF = new File(destPath);
boolean b = oldF.renameTo(newF);
ALog.d(TAG, String.format("剪裁文件消耗:%smsfileSize%sthreadId%s",
(System.currentTimeMillis() - stime), newF.length(),
getThreadConfig().record.threadId));
return b;
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (fis != null) {
fis.close();
}
if (fos != null) {
fos.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return false;
}
}
}