package com.aoleyun.sn.utils; import android.content.Context; import android.os.Build; import android.provider.Settings; import android.text.TextUtils; import android.util.Log; import android.webkit.WebSettings; import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.HashSet; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; import io.reactivex.rxjava3.core.Observable; import io.reactivex.rxjava3.core.ObservableEmitter; import io.reactivex.rxjava3.core.ObservableOnSubscribe; import io.reactivex.rxjava3.core.Observer; import io.reactivex.rxjava3.disposables.Disposable; import io.reactivex.rxjava3.schedulers.Schedulers; import okhttp3.Call; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; public class URLUtils { private static final String TAG = "URLUtils"; private Context mContext; private HashSet baseURLList = new HashSet<>(); private HashSet nohttpURLList = new HashSet<>(); private HashSet whiteIPList = new HashSet<>(); private long time1; private long time2; public URLUtils(Context context) { this.mContext = context; } public void setBrowserWhiteList() { time1 = System.currentTimeMillis(); //会写入两次,第一次是不带前缀的,第二次会把实际的地址写入进去 String oldwhiteList = Settings.System.getString(mContext.getContentResolver(), "DeselectBrowserArray"); String oldWhiteIP = (String) SPUtils.get(mContext, "white_ip", ""); if (TextUtils.isEmpty(oldwhiteList) && TextUtils.isEmpty(oldWhiteIP)) { Log.e(TAG, "getBrowserWhiteList: " + "oldwhiteList is empty"); Settings.System.putString(mContext.getContentResolver(), "DeselectBrowserArray", "Invalid"); } else { String white_ip = (String) SPUtils.get(mContext, "white_ip", ""); if (!TextUtils.isEmpty(white_ip)) { whiteIPList = new HashSet<>(Arrays.asList(white_ip.split(","))); } whiteIPList.removeIf(TextUtils::isEmpty); Log.e(TAG, "getBrowserWhiteList: " + "oldwhiteList is : " + oldwhiteList); if (TextUtils.isEmpty(oldwhiteList)) { return; } HashSet URLList = new HashSet<>(Arrays.asList(oldwhiteList.trim().split(","))); URLList.removeIf(TextUtils::isEmpty); if (URLList.size() == 0 && whiteIPList.size() == 0) { Log.e(TAG, "setBrowserWhiteList: URLList set is empty , set Invalid"); Settings.System.putString(mContext.getContentResolver(), "DeselectBrowserArray", "Invalid"); return; } Log.e(TAG, "setBrowserWhiteList: " + URLList); Observable.create(new ObservableOnSubscribe() { @Override public void subscribe(ObservableEmitter emitter) throws Exception { baseURLList.clear(); for (String url : URLList) { Log.e(TAG, "subscribe: " + url); if (TextUtils.isEmpty(url)) { continue; } // if (url.startsWith("http://")) { // String noHttp = url.substring(7); // if (!baseURLList.contains(noHttp)) { // baseURLList.add(noHttp); // } // } else if (url.startsWith("https://")) { // String noHttps = url.substring(8); // if (!baseURLList.contains(noHttps)) { // baseURLList.add(noHttps); // } // }else { // } if (haveDomain(url)) { nohttpURLList.add("m." + getDomain(url)); } // nohttpURLList.add(getDomain(url)); if (!url.startsWith("http")) { nohttpURLList.add(url); String httpsUrl = "https://" + url; String httpUrl = "http://" + url; emitter.onNext(getOkHttpURL(httpsUrl)); emitter.onNext(getOkHttpURL(httpUrl)); baseURLList.add(httpsUrl); String pattern = "(http|https)://(www.)?(\\w+(\\.)?)+"; Pattern r = Pattern.compile(pattern); Matcher m = r.matcher(httpsUrl); while (m.find()) { Log.e(TAG, "matcher1: " + m.group()); baseURLList.add(m.group()); } } else { if (url.endsWith("/")) { baseURLList.add(url.substring(0, url.length() - 1)); } //临时 if (baseURLList.size() != 0) { Settings.System.putString(mContext.getContentResolver(), "DeselectBrowserArray", String.join(",", baseURLList)); } Log.e(TAG, "subscribe: baseURLList: " + baseURLList); Log.e(TAG, "subscribe: DeselectBrowserArray: " + Settings.System.getString(mContext.getContentResolver(), "DeselectBrowserArray")); String pattern = "[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+\\.?"; Pattern r = Pattern.compile(pattern); Matcher m = r.matcher(url); if (m.find()) { nohttpURLList.add(m.group()); Log.e(TAG, "matcher2: " + m.group()); String httpsUrl = "https://" + m.group(); String httpUrl = "http://" + m.group(); emitter.onNext(getOkHttpURL(httpsUrl)); emitter.onNext(getOkHttpURL(httpUrl)); } } // if (url.startsWith("www.")) {//前缀为www的顶级域名 // String nowww = url.substring(4); // baseURLList.add(nowww); // baseURLList.add("http://" + nowww); // emitter.onNext(getOkHttpURL("http://" + nowww)); // baseURLList.add("https://" + nowww); // emitter.onNext(getOkHttpURL("https://" + nowww)); // Log.e(TAG, "subscribe: nowww: " + nowww); // baseURLList.add("http://" + url); // emitter.onNext(getOkHttpURL("http://" + url)); // baseURLList.add("https://" + url); // emitter.onNext(getOkHttpURL("https://" + url)); // } else if (getCount(url, ".") == 1 && getCount(url, ":") == 0) {//不带www的顶级域名 // baseURLList.add("http://www." + url); // emitter.onNext(getOkHttpURL("http://www." + url)); // baseURLList.add("https://www." + url); // emitter.onNext(getOkHttpURL("https://www." + url)); // baseURLList.add("http://" + url); // emitter.onNext(getOkHttpURL("http://" + url)); // baseURLList.add("https://" + url); // emitter.onNext(getOkHttpURL("https://" + url)); // } else if (!url.startsWith("http://") && !url.startsWith("https://")) {//没有前缀的网址 // baseURLList.add("http://" + url); // emitter.onNext(getOkHttpURL("http://" + url)); // baseURLList.add("https://" + url); // emitter.onNext(getOkHttpURL("https://" + url)); // } else { //// Log.e(TAG, "subscribe: 0:" + url); // baseURLList.add(url); // if (!TextUtils.isEmpty(url)) { // emitter.onNext(getOkHttpURL(url)); // } // } // Log.e(TAG, "subscribe: " + url); // emitter.onNext(getOkHttpURL(url)); } emitter.onComplete(); } }).subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .subscribe(new Observer() { @Override public void onSubscribe(Disposable d) { } @Override public void onNext(String s) { Log.e(TAG, "onNext: " + s); baseURLList.add(s); // if (s.startsWith("http://")) { // String noHttp = s.substring(7); // Log.e(TAG, "onNext: noHttp: " + noHttp); // baseURLList.add(noHttp); // } // if (s.startsWith("https://")) { // String noHttps = s.substring(8); // Log.e(TAG, "onNext: noHttps: " + noHttps); // baseURLList.add(noHttps); // } } @Override public void onError(Throwable e) { Log.e(TAG, "onError: " + e.getMessage()); } @Override public void onComplete() { Log.i(TAG, "onComplete: " + baseURLList); baseURLList.addAll(nohttpURLList); baseURLList.addAll(whiteIPList); baseURLList.removeIf(TextUtils::isEmpty); String DeselectBrowserArray = String.join(",", baseURLList); Log.e(TAG, "onComplete: " + "white list: " + DeselectBrowserArray); boolean write = Settings.System.putString(mContext.getContentResolver(), "DeselectBrowserArray", DeselectBrowserArray); Log.i(TAG, "onComplete: " + "write :" + write + " time = " + (System.currentTimeMillis() - time1)); } }); } } private HashSet blackList = new HashSet<>(); private HashSet blackIPList = new HashSet<>(); public void setBrowserBlackList() { String qch_webblack_url = Settings.System.getString(mContext.getContentResolver(), "qch_webblack_url"); Log.e(TAG, "setBrowserBlackList: qch_webblack_url = " + qch_webblack_url); if (TextUtils.isEmpty(qch_webblack_url)) { Settings.System.putString(mContext.getContentResolver(), "qch_webblack_url", "Invalid"); } else { String black_ip = (String) SPUtils.get(mContext, "black_ip", ""); if (!TextUtils.isEmpty(black_ip)) { blackIPList = new HashSet<>(Arrays.asList(black_ip.split(","))); } blackIPList.removeIf(TextUtils::isEmpty); HashSet URLList = new HashSet<>(Arrays.asList(qch_webblack_url.trim().split(","))); for (String url : URLList) { if (TextUtils.isEmpty(url)) { continue; } Log.e(TAG, "setBrowserBlackList: url: " + url); if (qch_webblack_url.startsWith("http://")) { blackList.add(url); blackList.add(url.replace("http://", "https://")); } else if (qch_webblack_url.startsWith("https://")) { blackList.add(url); blackList.add(url.replace("https://", "http://")); } else { blackList.add("http://" + url); blackList.add("https://" + url); } } blackList.addAll(blackIPList); blackList.removeIf(TextUtils::isEmpty); Log.e(TAG, "setBrowserBlackList: blackList: " + blackList); boolean write = Settings.System.putString(mContext.getContentResolver(), "qch_webblack_url", String.join(",", blackList)); Log.e(TAG, "setBrowserBlackList: write: " + write); } } private String getOkHttpURL(String URL) { if (!URL.startsWith("http")) { return ""; } OkHttpClient okHttpClient = new OkHttpClient.Builder() .connectTimeout(1, TimeUnit.SECONDS) .writeTimeout(1, TimeUnit.SECONDS) .readTimeout(1, TimeUnit.SECONDS) .build(); final Request request = new Request.Builder() .url(URL) .removeHeader("User-Agent") .addHeader("User-Agent", getUserAgent()) .get()//默认就是GET请求,可以不写 .build(); Call call = okHttpClient.newCall(request); // call.enqueue(new Callback() { // @Override // public void onFailure(Call call, IOException e) { // Log.e(TAG, "onFailure: "); // } // // @Override // public void onResponse(Call call, Response response) throws IOException { // Log.e(TAG, "onResponse: " + getIP(response.request().url().uri())); // } // }); try { Response response = call.execute(); if (response.isSuccessful()) { Log.e(TAG, "getOkHttpURL: " + response.request().url().toString()); return getIP(response.request().url().uri()).toString(); } else { return ""; } } catch (IOException e) { e.printStackTrace(); Log.w(TAG, "getOkHttpURL: " + e.getMessage() + " : " + URL); return ""; } } //https://blog.csdn.net/yong472727322/article/details/73321935 //https://blog.csdn.net/liehuo123/article/details/81509486?utm_medium=distribute.pc_relevant.none-task-blog-baidujs_baidulandingword-1&spm=1001.2101.3001.4242 //https://www.cnblogs.com/breakdown/archive/2012/09/17/2689101.html /** * 获取主机名 * * @param url 网址 * @return */ //https://blog.csdn.net/smallnetvisitor/article/details/84516347 public String getDomain(String url) { String regexStr = "(?<=//|)((\\w)+(\\.cn|\\.com.cn|\\.org.cn|\\.com|\\.net|\\.org|\\.cc|\\.biz|\\.uk|\\.info|\\.in|\\.eu))+"; Pattern p = Pattern.compile(regexStr); Matcher m = p.matcher(url); String domainVal = ""; if (m.find()) { domainVal = m.group(); } return domainVal; } public boolean haveDomain(String url) { String regexStr = "(?<=//|)((\\w)+(\\.cn|\\.com.cn|\\.org.cn|\\.com|\\.net|\\.org|\\.cc|\\.biz|\\.uk|\\.info|\\.in|\\.eu))+"; Pattern p = Pattern.compile(regexStr); Matcher m = p.matcher(url); return m.find(); } /** * Gets ip. * 通过url获取到域名 * https://www.cnblogs.com/shizhijie/p/8277092.html * * @param url the url * @return the ip */ public String getIP(String url) { //使用正则表达式过滤, String re = "((http|ftp|https)://)(([a-zA-Z0-9._-]+)|([0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}))(([a-zA-Z]{2,6})|(:[0-9]{1,4})?)"; String str = ""; // 编译正则表达式 Pattern pattern = Pattern.compile(re); // 忽略大小写的写法 // Pattern pat = Pattern.compile(regEx, Pattern.CASE_INSENSITIVE); Matcher matcher = pattern.matcher(url); //若url==http://127.0.0.1:9040或www.baidu.com的,正则表达式表示匹配 if (matcher.matches()) { str = url; } else { String[] split2 = url.split(re); if (split2.length > 1) { String substring = url.substring(0, url.length() - split2[1].length()); str = substring; } else { str = split2[0]; } } return str; } private URI getIP(URI uri) { URI effectiveURI = null; try { // URI(String scheme, String userInfo, String host, int port, String // path, String query,String fragment) effectiveURI = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), null, null, null); } catch (Throwable var4) { effectiveURI = null; } return effectiveURI; } /** * @return 获取浏览器的UA */ private String getUserAgent() { String userAgent = ""; if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { try { userAgent = WebSettings.getDefaultUserAgent(mContext); //需要hook webview } catch (Exception e) { Log.e(TAG, "getUserAgent: " + e.getMessage()); userAgent = System.getProperty("http.agent"); } } else { userAgent = System.getProperty("http.agent"); } StringBuffer sb = new StringBuffer(); for (int i = 0, length = userAgent.length(); i < length; i++) { char c = userAgent.charAt(i); if (c <= '\u001f' || c >= '\u007f') { sb.append(String.format("\\u%04x", (int) c)); } else { sb.append(c); } } return sb.toString(); } public int getCount(String str, String key) { if (str == null || key == null || "".equals(str.trim()) || "".equals(key.trim())) { return 0; } int count = 0; int index = 0; while ((index = str.indexOf(key, index)) != -1) { index = index + key.length(); count++; } return count; } }