add screen capture and color searching

This commit is contained in:
hyb1996
2017-05-22 15:41:11 +08:00
parent b3fad201f7
commit 302879333a
45 changed files with 1547 additions and 92 deletions

View File

@@ -84,6 +84,9 @@
<activity android:name=".ui.help.LocalWebViewActivity"/>
<activity android:name=".external.tasker.TaskerScriptEditActivity"/>
<activity android:name=".ui.edit.ViewSampleActivity"/>
<activity
android:name=".autojs.api.ScreenCaptureRequestActivity"
android:taskAffinity="com.stardust.scriptdroid.autojs.api.ScreenCaptureRequestActivity"/>
<activity
android:name=".ui.error.IssueReporterActivity"

View File

@@ -0,0 +1,3 @@
### requestScreenCapture(\[width, height\])
* width \<Number\> 可选参数

View File

@@ -5,7 +5,8 @@ var dialogs = {};
dialogs.rawInput = function(title, prefill){
prefill = prefill || "";
return String(__runtime__.dialogs.rawInput(title, prefill));
var s = __runtime__.dialogs.rawInput(title, prefill);
return s ? String(s) : null;
};
dialogs.input = function(title, prefill){

View File

@@ -0,0 +1,17 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
var img = captureScreen();
//0xffffff为白色
toastLog("开始找色");
//指定在位置(90, 220)宽高为900*1000的区域找色。
//0xff00cc是编辑器的深粉红色字体(字符串)颜色
var point = findColorInRegion(img, 0xff00cc, 90, 220, 900, 1000);
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -0,0 +1,18 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
var img = captureScreen();
//0xffffff为白色
toastLog("开始找色");
//指定在位置(90, 220)宽高为900*1000的区域找色。
//0xff00cc是编辑器的深粉红色字体(字符串)颜色
var point = findColor(img, 0xff00cc, {
region: [90, 220, 900, 1000],
threads: 8
});
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -0,0 +1,19 @@
//减少截图分辨率以提高速度
if(!requestScreenCapture(640, 960)){
toast("请求截图失败");
stop();
}
var img = captureScreen();
toastLog("开始找色");
//0x02b902为输入法绿色字体的颜色
var point = findColor(img, 0x02b902, {
//指定用8个线程找色
threads: 8
});
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -0,0 +1,6 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
var img = captureScreen();
images.saveImage(img, "/sdcard/1.png");

View File

@@ -0,0 +1,6 @@
//指定截图分辨率为 640×960
if(!requestScreenCapture(640, 960)){
toast("请求截图失败");
stop();
}
captureScreen("/sdcard/1.png");

View File

@@ -0,0 +1,16 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
launchApp("QQ");
sleep(2000);
var img = captureScreen();
toastLog("开始找色");
var point = findColor(img, 0xf64d30);
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -0,0 +1,22 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
var img = captureScreen();
//0xffffff为白色
toastLog("开始找色");
var point = findColor(img, 0xffffff, {
//指定算法为rgb+,更默认算法rgb更准确,但时间更久
algorithm: "rgb+",
//指定颜色临界值为16
threshold: 16,
//指定用8个线程找色
threads: 8
});
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -0,0 +1,15 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
var img = captureScreen();
//0x9966ff为编辑器紫色字体的颜色
toastLog("开始找色");
var point = findColor(img, 0x9966ff);
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -0,0 +1,16 @@
if(!requestScreenCapture()){
toast("请求截图失败");
stop();
}
var img = captureScreen();
toastLog("开始找色");
//0x006699为编辑器蓝色字体(var)的颜色
//找到颜色与0x006699完全相等的颜色
var point = findColorEquals(img, 0x006699);
if(point){
toastLog("x = " + point.x + ", y = " + point.y);
}else{
toastLog("没有找到");
}

View File

@@ -1,53 +0,0 @@
package com.stardust.app;
import android.content.Intent;
import android.util.SparseArray;
import java.util.ArrayList;
import java.util.List;
/**
* Created by Stardust on 2017/3/5.
*/
public interface OnActivityResultDelegate {
void onActivityResult(int requestCode, int resultCode, Intent data);
interface DelegateHost {
Mediator getDelegateManger();
}
class Mediator implements OnActivityResultDelegate {
private SparseArray<OnActivityResultDelegate> mSpecialDelegate = new SparseArray<>();
private List<OnActivityResultDelegate> mDelegates = new ArrayList<>();
public void onActivityResult(int requestCode, int resultCode, Intent data) {
OnActivityResultDelegate delegate = mSpecialDelegate.get(requestCode);
if (delegate != null) {
delegate.onActivityResult(requestCode, resultCode, data);
}
for (OnActivityResultDelegate d : mDelegates) {
d.onActivityResult(requestCode, resultCode, data);
}
}
public void addDelegate(OnActivityResultDelegate delegate) {
mDelegates.add(delegate);
}
public void addDelegate(int requestCode, OnActivityResultDelegate delegate) {
mSpecialDelegate.put(requestCode, delegate);
}
public void removeDelegate(OnActivityResultDelegate delegate) {
if (mDelegates.remove(delegate)) {
// TODO: 2017/3/16 优化
mSpecialDelegate.removeAt(mSpecialDelegate.indexOfValue(delegate));
}
}
}
}

View File

@@ -1,6 +1,7 @@
package com.stardust.scriptdroid;
import android.app.Activity;
import android.app.Application;
import android.content.IntentFilter;
import android.os.Bundle;
import android.support.multidex.MultiDexApplication;
@@ -79,26 +80,7 @@ public class App extends MultiDexApplication {
}
private void registerActivityLifecycleCallback() {
registerActivityLifecycleCallbacks(new SimpleActivityLifecycleCallbacks() {
@Override
public void onActivityCreated(Activity activity, Bundle savedInstanceState) {
super.onActivityCreated(activity, savedInstanceState);
AutoJs.getInstance().getAppUtils().setCurrentActivity(activity);
}
@Override
public void onActivityPaused(Activity activity) {
AutoJs.getInstance().getAppUtils().setCurrentActivity(null);
}
@Override
public void onActivityResumed(Activity activity) {
ScreenMetrics.initIfNeeded(activity);
AutoJs.getInstance().getAppUtils().setCurrentActivity(activity);
}
});
}
public static String getResString(int id) {

View File

@@ -1,9 +1,15 @@
package com.stardust.scriptdroid.autojs;
import android.accessibilityservice.AccessibilityService;
import android.app.Activity;
import android.content.Context;
import android.os.Build;
import android.os.Bundle;
import android.support.annotation.Nullable;
import android.support.annotation.RequiresApi;
import com.stardust.app.OnActivityResultDelegate;
import com.stardust.app.SimpleActivityLifecycleCallbacks;
import com.stardust.autojs.ScriptEngineService;
import com.stardust.autojs.ScriptEngineServiceBuilder;
import com.stardust.autojs.engine.RhinoJavaScriptEngineManager;
@@ -12,6 +18,7 @@ import com.stardust.autojs.runtime.AccessibilityBridge;
import com.stardust.autojs.runtime.ScriptStopException;
import com.stardust.autojs.runtime.api.AbstractShell;
import com.stardust.autojs.runtime.api.AppUtils;
import com.stardust.autojs.runtime.api.image.ScreenCaptureRequester;
import com.stardust.automator.AccessibilityEventCommandHost;
import com.stardust.automator.simple_action.SimpleActionPerformHost;
import com.stardust.pio.PFile;
@@ -19,8 +26,10 @@ import com.stardust.pio.UncheckedIOException;
import com.stardust.scriptdroid.App;
import com.stardust.scriptdroid.Pref;
import com.stardust.scriptdroid.R;
import com.stardust.scriptdroid.autojs.api.ScreenCaptureRequestActivity;
import com.stardust.scriptdroid.autojs.api.Shell;
import com.stardust.scriptdroid.ui.console.StardustConsole;
import com.stardust.util.ScreenMetrics;
import com.stardust.util.Supplier;
import com.stardust.util.UiHandler;
import com.stardust.view.accessibility.AccessibilityInfoProvider;
@@ -59,6 +68,23 @@ public class AutoJs implements AccessibilityBridge {
private final AccessibilityInfoProvider mAccessibilityInfoProvider;
private final UiHandler mUiHandler;
private final AppUtils mAppUtils;
private final ScreenCaptureRequester mScreenCaptureRequester = new ScreenCaptureRequester.AbstractScreenCaptureRequester() {
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
@Override
public void request() {
Activity activity = mAppUtils.getCurrentActivity();
if (activity instanceof OnActivityResultDelegate.DelegateHost) {
ScreenCaptureRequester requester = new ActivityScreenCaptureRequester(
((OnActivityResultDelegate.DelegateHost) activity).getOnActivityResultDelegateMediator(), activity);
requester.setOnActivityResultCallback(mCallback);
requester.request();
} else {
ScreenCaptureRequestActivity.request(mUiHandler.getContext(), mCallback);
}
}
};
private AutoJs(final Context context) {
@@ -75,8 +101,8 @@ public class AutoJs implements AccessibilityBridge {
@Override
public com.stardust.autojs.runtime.ScriptRuntime get() {
return new ScriptRuntime.Builder()
.setAppUtils(mAppUtils)
.setConsole(new StardustConsole(mUiHandler))
.setScreenCaptureRequester(mScreenCaptureRequester)
.setAccessibilityBridge(AutoJs.this)
.setUiHandler(mUiHandler)
.setShellSupplier(new Supplier<AbstractShell>() {
@@ -90,6 +116,28 @@ public class AutoJs implements AccessibilityBridge {
.build();
addAccessibilityServiceDelegates();
mScriptEngineService.registerGlobalScriptExecutionListener(new ScriptExecutionGlobalListener());
registerActivityLifecycleCallbacks();
}
private void registerActivityLifecycleCallbacks() {
App.getApp().registerActivityLifecycleCallbacks(new SimpleActivityLifecycleCallbacks() {
@Override
public void onActivityCreated(Activity activity, Bundle savedInstanceState) {
ScreenMetrics.initIfNeeded(activity);
mAppUtils.setCurrentActivity(activity);
}
@Override
public void onActivityPaused(Activity activity) {
mAppUtils.setCurrentActivity(null);
}
@Override
public void onActivityResumed(Activity activity) {
mAppUtils.setCurrentActivity(activity);
}
});
}
private ScriptEngineManager createScriptEngineManager(Context context) {

View File

@@ -7,6 +7,7 @@ import android.view.ContextThemeWrapper;
import com.afollestad.materialdialogs.MaterialDialog;
import com.stardust.autojs.runtime.ScriptInterface;
import com.stardust.autojs.runtime.ScriptInterruptedException;
import com.stardust.autojs.runtime.api.AppUtils;
import com.stardust.concurrent.VolatileBox;
import com.stardust.scriptdroid.R;
@@ -35,7 +36,7 @@ public class Dialogs {
.input(null, prefill, true, result)
.title(title)
.show();
return result.blockedGet();
return result.blockedGetOrThrow(ScriptInterruptedException.class);
}
@ScriptInterface
@@ -49,7 +50,7 @@ public class Dialogs {
builder.content(content);
}
builder.show();
lock.blockedGet();
lock.blockedGetOrThrow(ScriptInterruptedException.class);
}
@ScriptInterface
@@ -65,7 +66,7 @@ public class Dialogs {
builder.content(content);
}
builder.show();
return result.blockedGet();
return result.blockedGetOrThrow(ScriptInterruptedException.class);
}
private Context getContext() {
@@ -83,7 +84,7 @@ public class Dialogs {
.title(title)
.items((CharSequence[]) items)
.show();
return result.blockedGet();
return result.blockedGetOrThrow(ScriptInterruptedException.class);
}
@ScriptInterface
@@ -95,7 +96,7 @@ public class Dialogs {
.positiveText(R.string.ok)
.items((CharSequence[]) items)
.show();
return result.blockedGet();
return result.blockedGetOrThrow(ScriptInterruptedException.class);
}
@ScriptInterface
@@ -107,7 +108,7 @@ public class Dialogs {
.positiveText(R.string.ok)
.items((CharSequence[]) items)
.show();
return ArrayUtils.unbox(result.blockedGet());
return ArrayUtils.unbox(result.blockedGetOrThrow(ScriptInterruptedException.class));
}

View File

@@ -0,0 +1,50 @@
package com.stardust.scriptdroid.autojs.api;
import android.content.Context;
import android.content.Intent;
import android.os.Build;
import android.os.Bundle;
import android.support.annotation.Nullable;
import android.support.annotation.RequiresApi;
import com.stardust.app.OnActivityResultDelegate;
import com.stardust.autojs.runtime.api.image.ScreenCaptureRequester;
import com.stardust.scriptdroid.ui.BaseActivity;
/**
* Created by Stardust on 2017/5/22.
*/
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public class ScreenCaptureRequestActivity extends BaseActivity {
private static ScreenCaptureRequester.Callback sCallback;
public static void request(Context context, ScreenCaptureRequester.Callback callback) {
if (sCallback != null) {
return;
}
sCallback = callback;
context.startActivity(new Intent(context, ScreenCaptureRequestActivity.class)
.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK));
}
private OnActivityResultDelegate.Mediator mOnActivityResultDelegateMediator = new OnActivityResultDelegate.Mediator();
@Override
protected void onCreate(@Nullable Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
ScreenCaptureRequester requester = new ScreenCaptureRequester.ActivityScreenCaptureRequester(mOnActivityResultDelegateMediator, this);
requester.setOnActivityResultCallback(sCallback);
requester.request();
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
mOnActivityResultDelegateMediator.onActivityResult(requestCode, resultCode, data);
finish();
sCallback = null;
}
}

View File

@@ -0,0 +1,83 @@
package com.stardust.scriptdroid.service;
import android.app.Activity;
import android.app.Service;
import android.content.Context;
import android.content.Intent;
import android.graphics.PixelFormat;
import android.hardware.display.DisplayManager;
import android.hardware.display.VirtualDisplay;
import android.media.ImageReader;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
import android.os.Build;
import android.os.IBinder;
import android.support.annotation.RequiresApi;
import android.util.DisplayMetrics;
import android.view.WindowManager;
/**
* Created by Stardust on 2017/1/19.
*/
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
public class MediaProjectionScreenCaptureService extends Service {
private ImageReader mImageReader;
private int mScreenWidth;
private int mScreenHeight;
private int mScreenDensity;
private MediaProjection mMediaProjection;
private VirtualDisplay mVirtualDisplay;
public static Intent mResultData;
@Override
public void onCreate() {
if (mResultData == null) {
}
WindowManager windowManager = (WindowManager) getSystemService(Context.WINDOW_SERVICE);
DisplayMetrics metrics = new DisplayMetrics();
windowManager.getDefaultDisplay().getMetrics(metrics);
mScreenDensity = metrics.densityDpi;
mScreenWidth = metrics.widthPixels;
mScreenHeight = metrics.heightPixels;
createImageReader();
startVirtualDisplay();
}
@Override
public IBinder onBind(Intent intent) {
return null;
}
private void createImageReader() {
mImageReader = ImageReader.newInstance(mScreenWidth, mScreenHeight, PixelFormat.RGBA_8888, 1);
}
public void startVirtualDisplay() {
mMediaProjection = getMediaProjectionManager().getMediaProjection(Activity.RESULT_OK, mResultData);
mVirtualDisplay = mMediaProjection.createVirtualDisplay("screen-mirror",
mScreenWidth, mScreenHeight, mScreenDensity, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
mImageReader.getSurface(), null, null);
}
private MediaProjectionManager getMediaProjectionManager() {
return (MediaProjectionManager) getSystemService(Context.MEDIA_PROJECTION_SERVICE);
}
@Override
public void onDestroy() {
if (mVirtualDisplay != null) {
mVirtualDisplay.release();
mVirtualDisplay = null;
}
if (mMediaProjection != null) {
mMediaProjection.stop();
mMediaProjection = null;
}
super.onDestroy();
}
}

View File

@@ -117,7 +117,6 @@ public class SQLiteStaticsStorage implements ScriptStaticsStorage {
private static class SQLiteOpenHelper extends android.database.sqlite.SQLiteOpenHelper {
SQLiteOpenHelper(Context context) {
super(context, DATABASE_NAME, null, VERSION);
}

View File

@@ -25,7 +25,7 @@ public class ImageSelector implements OnActivityResultDelegate {
private Activity mActivity;
private ImageSelectorCallback mCallback;
public ImageSelector(Activity activity, Mediator mediator, ImageSelectorCallback callback) {
public ImageSelector(Activity activity, OnActivityResultDelegate.Mediator mediator, ImageSelectorCallback callback) {
mediator.addDelegate(REQUEST_CODE, this);
mActivity = activity;
mCallback = callback;

View File

@@ -21,10 +21,10 @@ import com.jecelyin.editor.v2.core.widget.TextView;
import com.jecelyin.editor.v2.ui.EditorDelegate;
import com.jecelyin.editor.v2.view.EditorView;
import com.jecelyin.editor.v2.view.menu.MenuDef;
import com.stardust.app.OnActivityResultDelegate;
import com.stardust.autojs.execution.ScriptExecution;
import com.stardust.autojs.script.FileScriptSource;
import com.stardust.autojs.script.JsBeautifier;
import com.stardust.autojs.script.StringScriptSource;
import com.stardust.scriptdroid.R;
import com.stardust.scriptdroid.autojs.AutoJs;
import com.stardust.scriptdroid.script.ScriptFile;
@@ -49,7 +49,7 @@ import java.io.File;
* Created by Stardust on 2017/1/29.
*/
public class EditActivity extends Editor920Activity {
public class EditActivity extends Editor920Activity implements OnActivityResultDelegate.DelegateHost {
public static class InputMethodEnhanceBarBridge implements InputMethodEnhanceBar.EditTextBridge {
@@ -76,6 +76,8 @@ public class EditActivity extends Editor920Activity {
public TextView getEditText() {
return mTextView;
}
}
@@ -102,6 +104,7 @@ public class EditActivity extends Editor920Activity {
private EditorDelegate mEditorDelegate;
private SparseArray<ToolbarMenuItem> mMenuMap;
private boolean mReadOnly = false;
private OnActivityResultDelegate.Mediator mActivityResultMediator = new OnActivityResultDelegate.Mediator();
private BroadcastReceiver mOnRunFinishedReceiver = new BroadcastReceiver() {
@Override
public void onReceive(Context context, Intent intent) {
@@ -359,6 +362,17 @@ public class EditActivity extends Editor920Activity {
.show();
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
mActivityResultMediator.onActivityResult(requestCode, resultCode, data);
}
@Override
public OnActivityResultDelegate.Mediator getOnActivityResultDelegateMediator() {
return mActivityResultMediator;
}
@Override
public void doCommand(Command command) {
mEditorDelegate.doCommand(command);

View File

@@ -15,6 +15,7 @@ import com.jecelyin.editor.v2.common.Command;
import com.jecelyin.editor.v2.ui.EditorDelegate;
import com.jecelyin.editor.v2.view.EditorView;
import com.jecelyin.editor.v2.view.menu.MenuDef;
import com.stardust.app.OnActivityResultDelegate;
import com.stardust.autojs.execution.ScriptExecution;
import com.stardust.autojs.script.StringScriptSource;
import com.stardust.scriptdroid.R;
@@ -42,7 +43,7 @@ import static com.stardust.scriptdroid.script.Scripts.EXTRA_EXCEPTION_MESSAGE;
* Created by Stardust on 2017/4/29.
*/
public class ViewSampleActivity extends Editor920Activity {
public class ViewSampleActivity extends Editor920Activity implements OnActivityResultDelegate.DelegateHost {
public static void view(Context context, Sample sample) {
@@ -56,6 +57,7 @@ public class ViewSampleActivity extends Editor920Activity {
private ScriptExecution mScriptExecution;
private EditorDelegate mEditorDelegate;
private SparseArray<ToolbarMenuItem> mMenuMap;
private OnActivityResultDelegate.Mediator mMediator = new OnActivityResultDelegate.Mediator();
private BroadcastReceiver mOnRunFinishedReceiver = new BroadcastReceiver() {
@Override
public void onReceive(Context context, Intent intent) {
@@ -185,4 +187,13 @@ public class ViewSampleActivity extends Editor920Activity {
}
}
@Override
public OnActivityResultDelegate.Mediator getOnActivityResultDelegateMediator() {
return mMediator;
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
mMediator.onActivityResult(requestCode, resultCode, data);
}
}

View File

@@ -62,10 +62,11 @@ import org.greenrobot.eventbus.Subscribe;
import java.io.IOException;
public class MainActivity extends BaseActivity {
public class MainActivity extends BaseActivity implements OnActivityResultDelegate.DelegateHost {
public static final String MESSAGE_CLEAR_BACKGROUND_SETTINGS = "MESSAGE_CLEAR_BACKGROUND_SETTINGS";
private static final String LOG_TAG = "MainActivity";
private static final String EXTRA_ACTION = "EXTRA_ACTION";
private static final String ACTION_ON_ACTION_RECORD_STOPPED = "ACTION_ON_ACTION_RECORD_STOPPED";
@@ -99,6 +100,11 @@ public class MainActivity extends BaseActivity {
showAnnunciationIfNeeded();
}
@Override
protected void onStart() {
super.onStart();
}
private void showAnnunciationIfNeeded() {
if (!Pref.shouldShowAnnunciation()) {
return;
@@ -406,4 +412,8 @@ public class MainActivity extends BaseActivity {
context.startActivity(intent);
}
@Override
public OnActivityResultDelegate.Mediator getOnActivityResultDelegateMediator() {
return mActivityResultMediator;
}
}

View File

@@ -5,7 +5,6 @@
android:accessibilityFlags="flagIncludeNotImportantViews|flagReportViewIds|flagRetrieveInteractiveWindows|flagRequestEnhancedWebAccessibility|flagRequestFilterKeyEvents"
android:canPerformGestures="true"
android:canRequestEnhancedWebAccessibility="true"
android:canRequestTouchExplorationMode="true"
android:canRetrieveWindowContent="true"
android:description="@string/text_accessibility_service_description"
android:notificationTimeout="100"/>